Compare commits
89 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f0c95ac0a9 | |||
| 0e392e18b4 | |||
| 8996f1ce06 | |||
| f60dbaf9c9 | |||
| 1a5a7aa9e1 | |||
| 326367ae4c | |||
| a7f9b0a22f | |||
| 0e525713b1 | |||
| f43fad02fb | |||
| eff3e378f3 | |||
| 5e32cef792 | |||
| 9898167f8d | |||
| 8ff4797041 | |||
| 4a553555d3 | |||
| 7a3202c996 | |||
| bd20ca86ab | |||
| 532cf95d59 | |||
| 366f8ce60d | |||
| 7612cb054a | |||
| 966d524dca | |||
| e84f1f1851 | |||
| a059f0502d | |||
| afd7173ba0 | |||
| c807bb2388 | |||
| aa4f9f5566 | |||
| 14786151e5 | |||
| a496862902 | |||
| df4f27ca2e | |||
| d72e0a347e | |||
| 6ed4b7d8ea | |||
| 31ebbea680 | |||
| d878134ebf | |||
| 50fb5aa387 | |||
| 55568d6892 | |||
| a4ae81c77c | |||
| 9da7104887 | |||
| cc161bf362 | |||
| de5377e5ac | |||
| 80b77b64eb | |||
| 6c912a3d71 | |||
| 2a0e77a754 | |||
| c6e368324a | |||
| 7b6c096bb7 | |||
| 03353a0aec | |||
| eb0e435a5a | |||
| 72f5d9d70c | |||
| be63e27c15 | |||
| 7633007a08 | |||
| 81ef0fe4c7 | |||
| 5f24bd920d | |||
| 8552f193de | |||
| de0d869c9f | |||
| 5536330aeb | |||
| d4657c389d | |||
| 3827235232 | |||
| fdd068feee | |||
| dfc0058d0d | |||
| 27aef84311 | |||
| 66b7677d2c | |||
| 5003514a3d | |||
| 4d90ead816 | |||
| 20a314e030 | |||
| b37a51ed1d | |||
| d8232340c3 | |||
| a356bb0c4e | |||
| 1c328ee3af | |||
| 5bf7962c04 | |||
| e06f653606 | |||
| 0462a7b62e | |||
| 9c2809c195 | |||
| fb32ca0a7d | |||
| 6ab702a818 | |||
| 550e7d435c | |||
| 776967e80d | |||
| 082a7fbcd1 | |||
| ff287cf67b | |||
| bddf36d52d | |||
| cf6cec9cab | |||
| d425839e57 | |||
| 4c661477d5 | |||
| f3f52f14a5 | |||
| d19ba3e305 | |||
| c627f41f53 | |||
| bcad0cd3da | |||
| 52660570c1 | |||
| 67f057ca1c | |||
| 35f61313e0 | |||
| 01e79e6993 | |||
| 1e3c4b545f |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -26,6 +26,10 @@ dist/
|
||||
*.egg-info/
|
||||
.eggs/
|
||||
|
||||
# Rust / maturin build artifacts
|
||||
myfsio_core/target/
|
||||
myfsio_core/Cargo.lock
|
||||
|
||||
# Local runtime artifacts
|
||||
logs/
|
||||
*.log
|
||||
|
||||
14
Dockerfile
14
Dockerfile
@@ -6,14 +6,26 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& apt-get install -y --no-install-recommends build-essential curl \
|
||||
&& curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN pip install --no-cache-dir maturin \
|
||||
&& cd myfsio_core \
|
||||
&& maturin build --release \
|
||||
&& pip install target/wheels/*.whl \
|
||||
&& cd .. \
|
||||
&& rm -rf myfsio_core/target \
|
||||
&& pip uninstall -y maturin \
|
||||
&& rustup self uninstall -y
|
||||
|
||||
RUN chmod +x docker-entrypoint.sh
|
||||
|
||||
RUN mkdir -p /app/data \
|
||||
|
||||
@@ -80,7 +80,7 @@ python run.py --mode api # API only (port 5000)
|
||||
python run.py --mode ui # UI only (port 5100)
|
||||
```
|
||||
|
||||
**Default Credentials:** `localadmin` / `localadmin`
|
||||
**Credentials:** Generated automatically on first run and printed to the console. If missed, check the IAM config file at `<STORAGE_ROOT>/.myfsio.sys/config/iam.json`.
|
||||
|
||||
- **Web Console:** http://127.0.0.1:5100/ui
|
||||
- **API Endpoint:** http://127.0.0.1:5000
|
||||
|
||||
285
app/__init__.py
285
app/__init__.py
@@ -1,20 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import html as html_module
|
||||
import itertools
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
from datetime import timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from flask import Flask, g, has_request_context, redirect, render_template, request, url_for
|
||||
from flask import Flask, Response, g, has_request_context, redirect, render_template, request, url_for
|
||||
from flask_cors import CORS
|
||||
from flask_wtf.csrf import CSRFError
|
||||
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||
|
||||
import io
|
||||
|
||||
from .access_logging import AccessLoggingService
|
||||
from .operation_metrics import OperationMetricsCollector, classify_endpoint
|
||||
from .compression import GzipMiddleware
|
||||
@@ -26,14 +31,77 @@ from .encryption import EncryptionManager
|
||||
from .extensions import limiter, csrf
|
||||
from .iam import IamService
|
||||
from .kms import KMSManager
|
||||
from .gc import GarbageCollector
|
||||
from .integrity import IntegrityChecker
|
||||
from .lifecycle import LifecycleManager
|
||||
from .notifications import NotificationService
|
||||
from .object_lock import ObjectLockService
|
||||
from .replication import ReplicationManager
|
||||
from .secret_store import EphemeralSecretStore
|
||||
from .site_registry import SiteRegistry, SiteInfo
|
||||
from .storage import ObjectStorage
|
||||
from .storage import ObjectStorage, StorageError
|
||||
from .version import get_version
|
||||
from .website_domains import WebsiteDomainStore
|
||||
|
||||
_request_counter = itertools.count(1)
|
||||
|
||||
|
||||
class _ChunkedTransferMiddleware:
|
||||
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
def __call__(self, environ, start_response):
|
||||
if environ.get("REQUEST_METHOD") not in ("PUT", "POST"):
|
||||
return self.app(environ, start_response)
|
||||
|
||||
transfer_encoding = environ.get("HTTP_TRANSFER_ENCODING", "")
|
||||
content_length = environ.get("CONTENT_LENGTH")
|
||||
|
||||
if "chunked" in transfer_encoding.lower():
|
||||
if content_length:
|
||||
del environ["HTTP_TRANSFER_ENCODING"]
|
||||
else:
|
||||
raw = environ.get("wsgi.input")
|
||||
if raw:
|
||||
try:
|
||||
if hasattr(raw, "seek"):
|
||||
raw.seek(0)
|
||||
body = raw.read()
|
||||
except Exception:
|
||||
body = b""
|
||||
if body:
|
||||
environ["wsgi.input"] = io.BytesIO(body)
|
||||
environ["CONTENT_LENGTH"] = str(len(body))
|
||||
del environ["HTTP_TRANSFER_ENCODING"]
|
||||
|
||||
content_length = environ.get("CONTENT_LENGTH")
|
||||
if not content_length or content_length == "0":
|
||||
sha256 = environ.get("HTTP_X_AMZ_CONTENT_SHA256", "")
|
||||
decoded_len = environ.get("HTTP_X_AMZ_DECODED_CONTENT_LENGTH", "")
|
||||
content_encoding = environ.get("HTTP_CONTENT_ENCODING", "")
|
||||
if ("STREAMING" in sha256.upper() or decoded_len
|
||||
or "aws-chunked" in content_encoding.lower()):
|
||||
raw = environ.get("wsgi.input")
|
||||
if raw:
|
||||
try:
|
||||
if hasattr(raw, "seek"):
|
||||
raw.seek(0)
|
||||
body = raw.read()
|
||||
except Exception:
|
||||
body = b""
|
||||
if body:
|
||||
environ["wsgi.input"] = io.BytesIO(body)
|
||||
environ["CONTENT_LENGTH"] = str(len(body))
|
||||
|
||||
raw = environ.get("wsgi.input")
|
||||
if raw and hasattr(raw, "seek"):
|
||||
try:
|
||||
raw.seek(0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return self.app(environ, start_response)
|
||||
|
||||
|
||||
def _migrate_config_file(active_path: Path, legacy_paths: List[Path]) -> Path:
|
||||
@@ -90,12 +158,20 @@ def create_app(
|
||||
app.config.setdefault("WTF_CSRF_ENABLED", False)
|
||||
|
||||
# Trust X-Forwarded-* headers from proxies
|
||||
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
|
||||
num_proxies = app.config.get("NUM_TRUSTED_PROXIES", 1)
|
||||
if num_proxies:
|
||||
if "NUM_TRUSTED_PROXIES" not in os.environ:
|
||||
logging.getLogger(__name__).warning(
|
||||
"NUM_TRUSTED_PROXIES not set, defaulting to 1. "
|
||||
"Set NUM_TRUSTED_PROXIES=0 if not behind a reverse proxy."
|
||||
)
|
||||
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=num_proxies, x_proto=num_proxies, x_host=num_proxies, x_prefix=num_proxies)
|
||||
|
||||
# Enable gzip compression for responses (10-20x smaller JSON payloads)
|
||||
if app.config.get("ENABLE_GZIP", True):
|
||||
app.wsgi_app = GzipMiddleware(app.wsgi_app, compression_level=6)
|
||||
|
||||
app.wsgi_app = _ChunkedTransferMiddleware(app.wsgi_app)
|
||||
|
||||
_configure_cors(app)
|
||||
_configure_logging(app)
|
||||
|
||||
@@ -104,7 +180,7 @@ def create_app(
|
||||
|
||||
storage = ObjectStorage(
|
||||
Path(app.config["STORAGE_ROOT"]),
|
||||
cache_ttl=app.config.get("OBJECT_CACHE_TTL", 5),
|
||||
cache_ttl=app.config.get("OBJECT_CACHE_TTL", 60),
|
||||
object_cache_max_size=app.config.get("OBJECT_CACHE_MAX_SIZE", 100),
|
||||
bucket_config_cache_ttl=app.config.get("BUCKET_CONFIG_CACHE_TTL_SECONDS", 30.0),
|
||||
object_key_max_length_bytes=app.config.get("OBJECT_KEY_MAX_LENGTH_BYTES", 1024),
|
||||
@@ -117,6 +193,7 @@ def create_app(
|
||||
Path(app.config["IAM_CONFIG"]),
|
||||
auth_max_attempts=app.config.get("AUTH_MAX_ATTEMPTS", 5),
|
||||
auth_lockout_minutes=app.config.get("AUTH_LOCKOUT_MINUTES", 15),
|
||||
encryption_key=app.config.get("SECRET_KEY"),
|
||||
)
|
||||
bucket_policies = BucketPolicyStore(Path(app.config["BUCKET_POLICY_PATH"]))
|
||||
secret_store = EphemeralSecretStore(default_ttl=app.config.get("SECRET_TTL_SECONDS", 300))
|
||||
@@ -207,6 +284,31 @@ def create_app(
|
||||
)
|
||||
lifecycle_manager.start()
|
||||
|
||||
gc_collector = None
|
||||
if app.config.get("GC_ENABLED", False):
|
||||
gc_collector = GarbageCollector(
|
||||
storage_root=storage_root,
|
||||
interval_hours=app.config.get("GC_INTERVAL_HOURS", 6.0),
|
||||
temp_file_max_age_hours=app.config.get("GC_TEMP_FILE_MAX_AGE_HOURS", 24.0),
|
||||
multipart_max_age_days=app.config.get("GC_MULTIPART_MAX_AGE_DAYS", 7),
|
||||
lock_file_max_age_hours=app.config.get("GC_LOCK_FILE_MAX_AGE_HOURS", 1.0),
|
||||
dry_run=app.config.get("GC_DRY_RUN", False),
|
||||
io_throttle_ms=app.config.get("GC_IO_THROTTLE_MS", 10),
|
||||
)
|
||||
gc_collector.start()
|
||||
|
||||
integrity_checker = None
|
||||
if app.config.get("INTEGRITY_ENABLED", False):
|
||||
integrity_checker = IntegrityChecker(
|
||||
storage_root=storage_root,
|
||||
interval_hours=app.config.get("INTEGRITY_INTERVAL_HOURS", 24.0),
|
||||
batch_size=app.config.get("INTEGRITY_BATCH_SIZE", 1000),
|
||||
auto_heal=app.config.get("INTEGRITY_AUTO_HEAL", False),
|
||||
dry_run=app.config.get("INTEGRITY_DRY_RUN", False),
|
||||
io_throttle_ms=app.config.get("INTEGRITY_IO_THROTTLE_MS", 10),
|
||||
)
|
||||
integrity_checker.start()
|
||||
|
||||
app.extensions["object_storage"] = storage
|
||||
app.extensions["iam"] = iam
|
||||
app.extensions["bucket_policies"] = bucket_policies
|
||||
@@ -218,11 +320,26 @@ def create_app(
|
||||
app.extensions["kms"] = kms_manager
|
||||
app.extensions["acl"] = acl_service
|
||||
app.extensions["lifecycle"] = lifecycle_manager
|
||||
app.extensions["gc"] = gc_collector
|
||||
app.extensions["integrity"] = integrity_checker
|
||||
app.extensions["object_lock"] = object_lock_service
|
||||
app.extensions["notifications"] = notification_service
|
||||
app.extensions["access_logging"] = access_logging_service
|
||||
app.extensions["site_registry"] = site_registry
|
||||
|
||||
website_domains_store = None
|
||||
if app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
website_domains_path = config_dir / "website_domains.json"
|
||||
website_domains_store = WebsiteDomainStore(website_domains_path)
|
||||
app.extensions["website_domains"] = website_domains_store
|
||||
|
||||
from .s3_client import S3ProxyClient
|
||||
api_base = app.config.get("API_BASE_URL") or "http://127.0.0.1:5000"
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(
|
||||
api_base_url=api_base,
|
||||
region=app.config.get("AWS_REGION", "us-east-1"),
|
||||
)
|
||||
|
||||
operation_metrics_collector = None
|
||||
if app.config.get("OPERATION_METRICS_ENABLED", False):
|
||||
operation_metrics_collector = OperationMetricsCollector(
|
||||
@@ -457,21 +574,164 @@ def _configure_logging(app: Flask) -> None:
|
||||
|
||||
@app.before_request
|
||||
def _log_request_start() -> None:
|
||||
g.request_id = uuid.uuid4().hex
|
||||
g.request_id = f"{os.getpid():x}{next(_request_counter):012x}"
|
||||
g.request_started_at = time.perf_counter()
|
||||
g.request_bytes_in = request.content_length or 0
|
||||
app.logger.info(
|
||||
"Request started",
|
||||
extra={"path": request.path, "method": request.method, "remote_addr": request.remote_addr},
|
||||
)
|
||||
|
||||
@app.before_request
|
||||
def _maybe_serve_website():
|
||||
if not app.config.get("WEBSITE_HOSTING_ENABLED"):
|
||||
return None
|
||||
if request.method not in {"GET", "HEAD"}:
|
||||
return None
|
||||
host = request.host
|
||||
if ":" in host:
|
||||
host = host.rsplit(":", 1)[0]
|
||||
host = host.lower()
|
||||
store = app.extensions.get("website_domains")
|
||||
if not store:
|
||||
return None
|
||||
bucket = store.get_bucket(host)
|
||||
if not bucket:
|
||||
return None
|
||||
storage = app.extensions["object_storage"]
|
||||
if not storage.bucket_exists(bucket):
|
||||
return _website_error_response(404, "Not Found")
|
||||
website_config = storage.get_bucket_website(bucket)
|
||||
if not website_config:
|
||||
return _website_error_response(404, "Not Found")
|
||||
index_doc = website_config.get("index_document", "index.html")
|
||||
error_doc = website_config.get("error_document")
|
||||
req_path = request.path.lstrip("/")
|
||||
if not req_path or req_path.endswith("/"):
|
||||
object_key = req_path + index_doc
|
||||
else:
|
||||
object_key = req_path
|
||||
try:
|
||||
obj_path = storage.get_object_path(bucket, object_key)
|
||||
except (StorageError, OSError):
|
||||
if object_key == req_path:
|
||||
try:
|
||||
obj_path = storage.get_object_path(bucket, req_path + "/" + index_doc)
|
||||
object_key = req_path + "/" + index_doc
|
||||
except (StorageError, OSError):
|
||||
return _serve_website_error(storage, bucket, error_doc, 404)
|
||||
else:
|
||||
return _serve_website_error(storage, bucket, error_doc, 404)
|
||||
content_type = mimetypes.guess_type(object_key)[0] or "application/octet-stream"
|
||||
is_encrypted = False
|
||||
try:
|
||||
metadata = storage.get_object_metadata(bucket, object_key)
|
||||
is_encrypted = "x-amz-server-side-encryption" in metadata
|
||||
except (StorageError, OSError):
|
||||
pass
|
||||
if is_encrypted and hasattr(storage, "get_object_data"):
|
||||
try:
|
||||
data, _ = storage.get_object_data(bucket, object_key)
|
||||
file_size = len(data)
|
||||
except (StorageError, OSError):
|
||||
return _website_error_response(500, "Internal Server Error")
|
||||
else:
|
||||
data = None
|
||||
try:
|
||||
stat = obj_path.stat()
|
||||
file_size = stat.st_size
|
||||
except OSError:
|
||||
return _website_error_response(500, "Internal Server Error")
|
||||
if request.method == "HEAD":
|
||||
response = Response(status=200)
|
||||
response.headers["Content-Length"] = file_size
|
||||
response.headers["Content-Type"] = content_type
|
||||
response.headers["Accept-Ranges"] = "bytes"
|
||||
return response
|
||||
from .s3_api import _parse_range_header
|
||||
range_header = request.headers.get("Range")
|
||||
if range_header:
|
||||
ranges = _parse_range_header(range_header, file_size)
|
||||
if ranges is None:
|
||||
return Response(status=416, headers={"Content-Range": f"bytes */{file_size}"})
|
||||
start, end = ranges[0]
|
||||
length = end - start + 1
|
||||
if data is not None:
|
||||
partial_data = data[start:end + 1]
|
||||
response = Response(partial_data, status=206, mimetype=content_type)
|
||||
else:
|
||||
def _stream_range(file_path, start_pos, length_to_read):
|
||||
with file_path.open("rb") as f:
|
||||
f.seek(start_pos)
|
||||
remaining = length_to_read
|
||||
while remaining > 0:
|
||||
chunk = f.read(min(262144, remaining))
|
||||
if not chunk:
|
||||
break
|
||||
remaining -= len(chunk)
|
||||
yield chunk
|
||||
response = Response(_stream_range(obj_path, start, length), status=206, mimetype=content_type, direct_passthrough=True)
|
||||
response.headers["Content-Range"] = f"bytes {start}-{end}/{file_size}"
|
||||
response.headers["Content-Length"] = length
|
||||
response.headers["Accept-Ranges"] = "bytes"
|
||||
return response
|
||||
if data is not None:
|
||||
response = Response(data, mimetype=content_type)
|
||||
response.headers["Content-Length"] = file_size
|
||||
response.headers["Accept-Ranges"] = "bytes"
|
||||
return response
|
||||
def _stream(file_path):
|
||||
with file_path.open("rb") as f:
|
||||
while True:
|
||||
chunk = f.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
response = Response(_stream(obj_path), mimetype=content_type, direct_passthrough=True)
|
||||
response.headers["Content-Length"] = file_size
|
||||
response.headers["Accept-Ranges"] = "bytes"
|
||||
return response
|
||||
|
||||
def _serve_website_error(storage, bucket, error_doc_key, status_code):
|
||||
if not error_doc_key:
|
||||
return _website_error_response(status_code, "Not Found" if status_code == 404 else "Error")
|
||||
try:
|
||||
obj_path = storage.get_object_path(bucket, error_doc_key)
|
||||
except (StorageError, OSError):
|
||||
return _website_error_response(status_code, "Not Found")
|
||||
content_type = mimetypes.guess_type(error_doc_key)[0] or "text/html"
|
||||
is_encrypted = False
|
||||
try:
|
||||
metadata = storage.get_object_metadata(bucket, error_doc_key)
|
||||
is_encrypted = "x-amz-server-side-encryption" in metadata
|
||||
except (StorageError, OSError):
|
||||
pass
|
||||
if is_encrypted and hasattr(storage, "get_object_data"):
|
||||
try:
|
||||
data, _ = storage.get_object_data(bucket, error_doc_key)
|
||||
response = Response(data, status=status_code, mimetype=content_type)
|
||||
response.headers["Content-Length"] = len(data)
|
||||
return response
|
||||
except (StorageError, OSError):
|
||||
return _website_error_response(status_code, "Not Found")
|
||||
try:
|
||||
data = obj_path.read_bytes()
|
||||
response = Response(data, status=status_code, mimetype=content_type)
|
||||
response.headers["Content-Length"] = len(data)
|
||||
return response
|
||||
except OSError:
|
||||
return _website_error_response(status_code, "Not Found")
|
||||
|
||||
def _website_error_response(status_code, message):
|
||||
safe_msg = html_module.escape(str(message))
|
||||
safe_code = html_module.escape(str(status_code))
|
||||
body = f"<html><head><title>{safe_code} {safe_msg}</title></head><body><h1>{safe_code} {safe_msg}</h1></body></html>"
|
||||
return Response(body, status=status_code, mimetype="text/html")
|
||||
|
||||
@app.after_request
|
||||
def _log_request_end(response):
|
||||
duration_ms = 0.0
|
||||
if hasattr(g, "request_started_at"):
|
||||
duration_ms = (time.perf_counter() - g.request_started_at) * 1000
|
||||
request_id = getattr(g, "request_id", uuid.uuid4().hex)
|
||||
request_id = getattr(g, "request_id", f"{os.getpid():x}{next(_request_counter):012x}")
|
||||
response.headers.setdefault("X-Request-ID", request_id)
|
||||
if app.logger.isEnabledFor(logging.INFO):
|
||||
app.logger.info(
|
||||
"Request completed",
|
||||
extra={
|
||||
@@ -481,6 +741,7 @@ def _configure_logging(app: Flask) -> None:
|
||||
},
|
||||
)
|
||||
response.headers["X-Request-Duration-ms"] = f"{duration_ms:.2f}"
|
||||
response.headers["Server"] = "MyFSIO"
|
||||
|
||||
operation_metrics = app.extensions.get("operation_metrics")
|
||||
if operation_metrics:
|
||||
|
||||
309
app/admin_api.py
309
app/admin_api.py
@@ -14,9 +14,12 @@ from flask import Blueprint, Response, current_app, jsonify, request
|
||||
|
||||
from .connections import ConnectionStore
|
||||
from .extensions import limiter
|
||||
from .gc import GarbageCollector
|
||||
from .integrity import IntegrityChecker
|
||||
from .iam import IamError, Principal
|
||||
from .replication import ReplicationManager
|
||||
from .site_registry import PeerSite, SiteInfo, SiteRegistry
|
||||
from .website_domains import WebsiteDomainStore, normalize_domain, is_valid_domain
|
||||
|
||||
|
||||
def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
|
||||
@@ -673,3 +676,309 @@ def check_bidirectional_status(site_id: str):
|
||||
result["is_fully_configured"] = len(error_issues) == 0 and len(local_bidir_rules) > 0
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
def _website_domains() -> WebsiteDomainStore:
|
||||
return current_app.extensions["website_domains"]
|
||||
|
||||
|
||||
def _storage():
|
||||
return current_app.extensions["object_storage"]
|
||||
|
||||
|
||||
def _require_iam_action(action: str):
|
||||
principal, error = _require_principal()
|
||||
if error:
|
||||
return None, error
|
||||
try:
|
||||
_iam().authorize(principal, None, action)
|
||||
return principal, None
|
||||
except IamError:
|
||||
return None, _json_error("AccessDenied", f"Requires {action} permission", 403)
|
||||
|
||||
|
||||
@admin_api_bp.route("/iam/users", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def iam_list_users():
|
||||
principal, error = _require_iam_action("iam:list_users")
|
||||
if error:
|
||||
return error
|
||||
return jsonify({"users": _iam().list_users()})
|
||||
|
||||
|
||||
@admin_api_bp.route("/iam/users/<identifier>", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def iam_get_user(identifier):
|
||||
principal, error = _require_iam_action("iam:get_user")
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
user_id = _iam().resolve_user_id(identifier)
|
||||
return jsonify(_iam().get_user_by_id(user_id))
|
||||
except IamError as exc:
|
||||
return _json_error("NotFound", str(exc), 404)
|
||||
|
||||
|
||||
@admin_api_bp.route("/iam/users/<identifier>/policies", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def iam_get_user_policies(identifier):
|
||||
principal, error = _require_iam_action("iam:get_policy")
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
return jsonify({"policies": _iam().get_user_policies(identifier)})
|
||||
except IamError as exc:
|
||||
return _json_error("NotFound", str(exc), 404)
|
||||
|
||||
|
||||
@admin_api_bp.route("/iam/users/<identifier>/keys", methods=["POST"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def iam_create_access_key(identifier):
|
||||
principal, error = _require_iam_action("iam:create_key")
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
result = _iam().create_access_key(identifier)
|
||||
logger.info("Access key created for %s by %s", identifier, principal.access_key)
|
||||
return jsonify(result), 201
|
||||
except IamError as exc:
|
||||
return _json_error("InvalidRequest", str(exc), 400)
|
||||
|
||||
|
||||
@admin_api_bp.route("/iam/users/<identifier>/keys/<access_key>", methods=["DELETE"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def iam_delete_access_key(identifier, access_key):
|
||||
principal, error = _require_iam_action("iam:delete_key")
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_iam().delete_access_key(access_key)
|
||||
logger.info("Access key %s deleted by %s", access_key, principal.access_key)
|
||||
return "", 204
|
||||
except IamError as exc:
|
||||
return _json_error("InvalidRequest", str(exc), 400)
|
||||
|
||||
|
||||
@admin_api_bp.route("/iam/users/<identifier>/disable", methods=["POST"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def iam_disable_user(identifier):
|
||||
principal, error = _require_iam_action("iam:disable_user")
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_iam().disable_user(identifier)
|
||||
logger.info("User %s disabled by %s", identifier, principal.access_key)
|
||||
return jsonify({"status": "disabled"})
|
||||
except IamError as exc:
|
||||
return _json_error("InvalidRequest", str(exc), 400)
|
||||
|
||||
|
||||
@admin_api_bp.route("/iam/users/<identifier>/enable", methods=["POST"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def iam_enable_user(identifier):
|
||||
principal, error = _require_iam_action("iam:disable_user")
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_iam().enable_user(identifier)
|
||||
logger.info("User %s enabled by %s", identifier, principal.access_key)
|
||||
return jsonify({"status": "enabled"})
|
||||
except IamError as exc:
|
||||
return _json_error("InvalidRequest", str(exc), 400)
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def list_website_domains():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
return jsonify(_website_domains().list_all())
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains", methods=["POST"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def create_website_domain():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
payload = request.get_json(silent=True) or {}
|
||||
domain = normalize_domain(payload.get("domain") or "")
|
||||
bucket = (payload.get("bucket") or "").strip()
|
||||
if not domain:
|
||||
return _json_error("ValidationError", "domain is required", 400)
|
||||
if not is_valid_domain(domain):
|
||||
return _json_error("ValidationError", f"Invalid domain: '{domain}'", 400)
|
||||
if not bucket:
|
||||
return _json_error("ValidationError", "bucket is required", 400)
|
||||
storage = _storage()
|
||||
if not storage.bucket_exists(bucket):
|
||||
return _json_error("NoSuchBucket", f"Bucket '{bucket}' does not exist", 404)
|
||||
store = _website_domains()
|
||||
existing = store.get_bucket(domain)
|
||||
if existing:
|
||||
return _json_error("Conflict", f"Domain '{domain}' is already mapped to bucket '{existing}'", 409)
|
||||
store.set_mapping(domain, bucket)
|
||||
logger.info("Website domain mapping created: %s -> %s", domain, bucket)
|
||||
return jsonify({"domain": domain, "bucket": bucket}), 201
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains/<domain>", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def get_website_domain(domain: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
domain = normalize_domain(domain)
|
||||
bucket = _website_domains().get_bucket(domain)
|
||||
if not bucket:
|
||||
return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404)
|
||||
return jsonify({"domain": domain, "bucket": bucket})
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains/<domain>", methods=["PUT"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def update_website_domain(domain: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
domain = normalize_domain(domain)
|
||||
payload = request.get_json(silent=True) or {}
|
||||
bucket = (payload.get("bucket") or "").strip()
|
||||
if not bucket:
|
||||
return _json_error("ValidationError", "bucket is required", 400)
|
||||
storage = _storage()
|
||||
if not storage.bucket_exists(bucket):
|
||||
return _json_error("NoSuchBucket", f"Bucket '{bucket}' does not exist", 404)
|
||||
store = _website_domains()
|
||||
if not store.get_bucket(domain):
|
||||
return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404)
|
||||
store.set_mapping(domain, bucket)
|
||||
logger.info("Website domain mapping updated: %s -> %s", domain, bucket)
|
||||
return jsonify({"domain": domain, "bucket": bucket})
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains/<domain>", methods=["DELETE"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def delete_website_domain(domain: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
domain = normalize_domain(domain)
|
||||
if not _website_domains().delete_mapping(domain):
|
||||
return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404)
|
||||
logger.info("Website domain mapping deleted: %s", domain)
|
||||
return Response(status=204)
|
||||
|
||||
|
||||
def _gc() -> Optional[GarbageCollector]:
|
||||
return current_app.extensions.get("gc")
|
||||
|
||||
|
||||
@admin_api_bp.route("/gc/status", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def gc_status():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
gc = _gc()
|
||||
if not gc:
|
||||
return jsonify({"enabled": False, "message": "GC is not enabled. Set GC_ENABLED=true to enable."})
|
||||
return jsonify(gc.get_status())
|
||||
|
||||
|
||||
@admin_api_bp.route("/gc/run", methods=["POST"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def gc_run_now():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
gc = _gc()
|
||||
if not gc:
|
||||
return _json_error("InvalidRequest", "GC is not enabled", 400)
|
||||
payload = request.get_json(silent=True) or {}
|
||||
started = gc.run_async(dry_run=payload.get("dry_run"))
|
||||
logger.info("GC manual run by %s", principal.access_key)
|
||||
if not started:
|
||||
return _json_error("Conflict", "GC is already in progress", 409)
|
||||
return jsonify({"status": "started"})
|
||||
|
||||
|
||||
@admin_api_bp.route("/gc/history", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def gc_history():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
gc = _gc()
|
||||
if not gc:
|
||||
return jsonify({"executions": []})
|
||||
limit = min(int(request.args.get("limit", 50)), 200)
|
||||
offset = int(request.args.get("offset", 0))
|
||||
records = gc.get_history(limit=limit, offset=offset)
|
||||
return jsonify({"executions": records})
|
||||
|
||||
|
||||
def _integrity() -> Optional[IntegrityChecker]:
|
||||
return current_app.extensions.get("integrity")
|
||||
|
||||
|
||||
@admin_api_bp.route("/integrity/status", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def integrity_status():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
checker = _integrity()
|
||||
if not checker:
|
||||
return jsonify({"enabled": False, "message": "Integrity checker is not enabled. Set INTEGRITY_ENABLED=true to enable."})
|
||||
return jsonify(checker.get_status())
|
||||
|
||||
|
||||
@admin_api_bp.route("/integrity/run", methods=["POST"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def integrity_run_now():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
checker = _integrity()
|
||||
if not checker:
|
||||
return _json_error("InvalidRequest", "Integrity checker is not enabled", 400)
|
||||
payload = request.get_json(silent=True) or {}
|
||||
override_dry_run = payload.get("dry_run")
|
||||
override_auto_heal = payload.get("auto_heal")
|
||||
started = checker.run_async(
|
||||
auto_heal=override_auto_heal if override_auto_heal is not None else None,
|
||||
dry_run=override_dry_run if override_dry_run is not None else None,
|
||||
)
|
||||
logger.info("Integrity manual run by %s", principal.access_key)
|
||||
if not started:
|
||||
return _json_error("Conflict", "A scan is already in progress", 409)
|
||||
return jsonify({"status": "started"})
|
||||
|
||||
|
||||
@admin_api_bp.route("/integrity/history", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def integrity_history():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
checker = _integrity()
|
||||
if not checker:
|
||||
return jsonify({"executions": []})
|
||||
limit = min(int(request.args.get("limit", 50)), 200)
|
||||
offset = int(request.args.get("offset", 0))
|
||||
records = checker.get_history(limit=limit, offset=offset)
|
||||
return jsonify({"executions": records})
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
@@ -75,7 +76,7 @@ def _evaluate_condition_operator(
|
||||
expected_null = condition_values[0].lower() in ("true", "1", "yes") if condition_values else True
|
||||
return is_null == expected_null
|
||||
|
||||
return True
|
||||
return False
|
||||
|
||||
ACTION_ALIASES = {
|
||||
"s3:listbucket": "list",
|
||||
@@ -268,7 +269,7 @@ class BucketPolicyStore:
|
||||
self._last_mtime = self._current_mtime()
|
||||
# Performance: Avoid stat() on every request
|
||||
self._last_stat_check = 0.0
|
||||
self._stat_check_interval = 1.0 # Only check mtime every 1 second
|
||||
self._stat_check_interval = float(os.environ.get("BUCKET_POLICY_STAT_CHECK_INTERVAL_SECONDS", "2.0"))
|
||||
|
||||
def maybe_reload(self) -> None:
|
||||
# Performance: Skip stat check if we checked recently
|
||||
|
||||
@@ -36,11 +36,11 @@ class GzipMiddleware:
|
||||
content_type = None
|
||||
content_length = None
|
||||
should_compress = False
|
||||
is_streaming = False
|
||||
passthrough = False
|
||||
exc_info_holder = [None]
|
||||
|
||||
def custom_start_response(status: str, headers: List[Tuple[str, str]], exc_info=None):
|
||||
nonlocal response_started, status_code, response_headers, content_type, content_length, should_compress, is_streaming
|
||||
nonlocal response_started, status_code, response_headers, content_type, content_length, should_compress, passthrough
|
||||
response_started = True
|
||||
status_code = int(status.split(' ', 1)[0])
|
||||
response_headers = list(headers)
|
||||
@@ -51,23 +51,29 @@ class GzipMiddleware:
|
||||
if name_lower == 'content-type':
|
||||
content_type = value.split(';')[0].strip().lower()
|
||||
elif name_lower == 'content-length':
|
||||
try:
|
||||
content_length = int(value)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
elif name_lower == 'content-encoding':
|
||||
should_compress = False
|
||||
passthrough = True
|
||||
return start_response(status, headers, exc_info)
|
||||
elif name_lower == 'x-stream-response':
|
||||
is_streaming = True
|
||||
passthrough = True
|
||||
return start_response(status, headers, exc_info)
|
||||
|
||||
if content_type and content_type in COMPRESSIBLE_MIMES:
|
||||
if content_length is None or content_length >= self.min_size:
|
||||
should_compress = True
|
||||
else:
|
||||
passthrough = True
|
||||
return start_response(status, headers, exc_info)
|
||||
|
||||
return None
|
||||
|
||||
app_iter = self.app(environ, custom_start_response)
|
||||
|
||||
if is_streaming:
|
||||
if passthrough:
|
||||
return app_iter
|
||||
|
||||
response_body = b''.join(app_iter)
|
||||
|
||||
@@ -25,7 +25,7 @@ def _calculate_auto_connection_limit() -> int:
|
||||
|
||||
|
||||
def _calculate_auto_backlog(connection_limit: int) -> int:
|
||||
return max(64, min(connection_limit * 2, 4096))
|
||||
return max(128, min(connection_limit * 2, 4096))
|
||||
|
||||
|
||||
def _validate_rate_limit(value: str) -> str:
|
||||
@@ -115,6 +115,7 @@ class AppConfig:
|
||||
server_connection_limit: int
|
||||
server_backlog: int
|
||||
server_channel_timeout: int
|
||||
server_max_buffer_size: int
|
||||
server_threads_auto: bool
|
||||
server_connection_limit_auto: bool
|
||||
server_backlog_auto: bool
|
||||
@@ -149,6 +150,20 @@ class AppConfig:
|
||||
num_trusted_proxies: int
|
||||
allowed_redirect_hosts: list[str]
|
||||
allow_internal_endpoints: bool
|
||||
website_hosting_enabled: bool
|
||||
gc_enabled: bool
|
||||
gc_interval_hours: float
|
||||
gc_temp_file_max_age_hours: float
|
||||
gc_multipart_max_age_days: int
|
||||
gc_lock_file_max_age_hours: float
|
||||
gc_dry_run: bool
|
||||
gc_io_throttle_ms: int
|
||||
integrity_enabled: bool
|
||||
integrity_interval_hours: float
|
||||
integrity_batch_size: int
|
||||
integrity_auto_heal: bool
|
||||
integrity_dry_run: bool
|
||||
integrity_io_throttle_ms: int
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, overrides: Optional[Dict[str, Any]] = None) -> "AppConfig":
|
||||
@@ -240,7 +255,7 @@ class AppConfig:
|
||||
cors_expose_headers = _csv(str(_get("CORS_EXPOSE_HEADERS", "*")), ["*"])
|
||||
session_lifetime_days = int(_get("SESSION_LIFETIME_DAYS", 30))
|
||||
bucket_stats_cache_ttl = int(_get("BUCKET_STATS_CACHE_TTL", 60))
|
||||
object_cache_ttl = int(_get("OBJECT_CACHE_TTL", 5))
|
||||
object_cache_ttl = int(_get("OBJECT_CACHE_TTL", 60))
|
||||
|
||||
encryption_enabled = str(_get("ENCRYPTION_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
encryption_keys_dir = storage_root / ".myfsio.sys" / "keys"
|
||||
@@ -281,6 +296,7 @@ class AppConfig:
|
||||
server_backlog_auto = False
|
||||
|
||||
server_channel_timeout = int(_get("SERVER_CHANNEL_TIMEOUT", 120))
|
||||
server_max_buffer_size = int(_get("SERVER_MAX_BUFFER_SIZE", 1024 * 1024 * 128))
|
||||
site_sync_enabled = str(_get("SITE_SYNC_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
site_sync_interval_seconds = int(_get("SITE_SYNC_INTERVAL_SECONDS", 60))
|
||||
site_sync_batch_size = int(_get("SITE_SYNC_BATCH_SIZE", 100))
|
||||
@@ -313,10 +329,24 @@ class AppConfig:
|
||||
site_region = str(_get("SITE_REGION", "us-east-1"))
|
||||
site_priority = int(_get("SITE_PRIORITY", 100))
|
||||
ratelimit_admin = _validate_rate_limit(str(_get("RATE_LIMIT_ADMIN", "60 per minute")))
|
||||
num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 0))
|
||||
num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 1))
|
||||
allowed_redirect_hosts_raw = _get("ALLOWED_REDIRECT_HOSTS", "")
|
||||
allowed_redirect_hosts = [h.strip() for h in str(allowed_redirect_hosts_raw).split(",") if h.strip()]
|
||||
allow_internal_endpoints = str(_get("ALLOW_INTERNAL_ENDPOINTS", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
website_hosting_enabled = str(_get("WEBSITE_HOSTING_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
gc_enabled = str(_get("GC_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
gc_interval_hours = float(_get("GC_INTERVAL_HOURS", 6.0))
|
||||
gc_temp_file_max_age_hours = float(_get("GC_TEMP_FILE_MAX_AGE_HOURS", 24.0))
|
||||
gc_multipart_max_age_days = int(_get("GC_MULTIPART_MAX_AGE_DAYS", 7))
|
||||
gc_lock_file_max_age_hours = float(_get("GC_LOCK_FILE_MAX_AGE_HOURS", 1.0))
|
||||
gc_dry_run = str(_get("GC_DRY_RUN", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
gc_io_throttle_ms = int(_get("GC_IO_THROTTLE_MS", 10))
|
||||
integrity_enabled = str(_get("INTEGRITY_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
integrity_interval_hours = float(_get("INTEGRITY_INTERVAL_HOURS", 24.0))
|
||||
integrity_batch_size = int(_get("INTEGRITY_BATCH_SIZE", 1000))
|
||||
integrity_auto_heal = str(_get("INTEGRITY_AUTO_HEAL", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
integrity_dry_run = str(_get("INTEGRITY_DRY_RUN", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
integrity_io_throttle_ms = int(_get("INTEGRITY_IO_THROTTLE_MS", 10))
|
||||
|
||||
return cls(storage_root=storage_root,
|
||||
max_upload_size=max_upload_size,
|
||||
@@ -370,6 +400,7 @@ class AppConfig:
|
||||
server_connection_limit=server_connection_limit,
|
||||
server_backlog=server_backlog,
|
||||
server_channel_timeout=server_channel_timeout,
|
||||
server_max_buffer_size=server_max_buffer_size,
|
||||
server_threads_auto=server_threads_auto,
|
||||
server_connection_limit_auto=server_connection_limit_auto,
|
||||
server_backlog_auto=server_backlog_auto,
|
||||
@@ -403,7 +434,21 @@ class AppConfig:
|
||||
ratelimit_admin=ratelimit_admin,
|
||||
num_trusted_proxies=num_trusted_proxies,
|
||||
allowed_redirect_hosts=allowed_redirect_hosts,
|
||||
allow_internal_endpoints=allow_internal_endpoints)
|
||||
allow_internal_endpoints=allow_internal_endpoints,
|
||||
website_hosting_enabled=website_hosting_enabled,
|
||||
gc_enabled=gc_enabled,
|
||||
gc_interval_hours=gc_interval_hours,
|
||||
gc_temp_file_max_age_hours=gc_temp_file_max_age_hours,
|
||||
gc_multipart_max_age_days=gc_multipart_max_age_days,
|
||||
gc_lock_file_max_age_hours=gc_lock_file_max_age_hours,
|
||||
gc_dry_run=gc_dry_run,
|
||||
gc_io_throttle_ms=gc_io_throttle_ms,
|
||||
integrity_enabled=integrity_enabled,
|
||||
integrity_interval_hours=integrity_interval_hours,
|
||||
integrity_batch_size=integrity_batch_size,
|
||||
integrity_auto_heal=integrity_auto_heal,
|
||||
integrity_dry_run=integrity_dry_run,
|
||||
integrity_io_throttle_ms=integrity_io_throttle_ms)
|
||||
|
||||
def validate_and_report(self) -> list[str]:
|
||||
"""Validate configuration and return a list of warnings/issues.
|
||||
@@ -468,10 +513,12 @@ class AppConfig:
|
||||
issues.append(f"CRITICAL: SERVER_THREADS={self.server_threads} is outside valid range (1-64). Server cannot start.")
|
||||
if not (10 <= self.server_connection_limit <= 1000):
|
||||
issues.append(f"CRITICAL: SERVER_CONNECTION_LIMIT={self.server_connection_limit} is outside valid range (10-1000). Server cannot start.")
|
||||
if not (64 <= self.server_backlog <= 4096):
|
||||
issues.append(f"CRITICAL: SERVER_BACKLOG={self.server_backlog} is outside valid range (64-4096). Server cannot start.")
|
||||
if not (128 <= self.server_backlog <= 4096):
|
||||
issues.append(f"CRITICAL: SERVER_BACKLOG={self.server_backlog} is outside valid range (128-4096). Server cannot start.")
|
||||
if not (10 <= self.server_channel_timeout <= 300):
|
||||
issues.append(f"CRITICAL: SERVER_CHANNEL_TIMEOUT={self.server_channel_timeout} is outside valid range (10-300). Server cannot start.")
|
||||
if self.server_max_buffer_size < 1024 * 1024:
|
||||
issues.append(f"WARNING: SERVER_MAX_BUFFER_SIZE={self.server_max_buffer_size} is less than 1MB. Large uploads will fail.")
|
||||
|
||||
if sys.platform != "win32":
|
||||
try:
|
||||
@@ -509,12 +556,15 @@ class AppConfig:
|
||||
print(f" ENCRYPTION: Enabled (Master key: {self.encryption_master_key_path})")
|
||||
if self.kms_enabled:
|
||||
print(f" KMS: Enabled (Keys: {self.kms_keys_path})")
|
||||
if self.website_hosting_enabled:
|
||||
print(f" WEBSITE_HOSTING: Enabled")
|
||||
def _auto(flag: bool) -> str:
|
||||
return " (auto)" if flag else ""
|
||||
print(f" SERVER_THREADS: {self.server_threads}{_auto(self.server_threads_auto)}")
|
||||
print(f" CONNECTION_LIMIT: {self.server_connection_limit}{_auto(self.server_connection_limit_auto)}")
|
||||
print(f" BACKLOG: {self.server_backlog}{_auto(self.server_backlog_auto)}")
|
||||
print(f" CHANNEL_TIMEOUT: {self.server_channel_timeout}s")
|
||||
print(f" MAX_BUFFER_SIZE: {self.server_max_buffer_size // (1024 * 1024)}MB")
|
||||
print("=" * 60)
|
||||
|
||||
issues = self.validate_and_report()
|
||||
@@ -580,6 +630,7 @@ class AppConfig:
|
||||
"SERVER_CONNECTION_LIMIT": self.server_connection_limit,
|
||||
"SERVER_BACKLOG": self.server_backlog,
|
||||
"SERVER_CHANNEL_TIMEOUT": self.server_channel_timeout,
|
||||
"SERVER_MAX_BUFFER_SIZE": self.server_max_buffer_size,
|
||||
"SITE_SYNC_ENABLED": self.site_sync_enabled,
|
||||
"SITE_SYNC_INTERVAL_SECONDS": self.site_sync_interval_seconds,
|
||||
"SITE_SYNC_BATCH_SIZE": self.site_sync_batch_size,
|
||||
@@ -611,4 +662,18 @@ class AppConfig:
|
||||
"NUM_TRUSTED_PROXIES": self.num_trusted_proxies,
|
||||
"ALLOWED_REDIRECT_HOSTS": self.allowed_redirect_hosts,
|
||||
"ALLOW_INTERNAL_ENDPOINTS": self.allow_internal_endpoints,
|
||||
"WEBSITE_HOSTING_ENABLED": self.website_hosting_enabled,
|
||||
"GC_ENABLED": self.gc_enabled,
|
||||
"GC_INTERVAL_HOURS": self.gc_interval_hours,
|
||||
"GC_TEMP_FILE_MAX_AGE_HOURS": self.gc_temp_file_max_age_hours,
|
||||
"GC_MULTIPART_MAX_AGE_DAYS": self.gc_multipart_max_age_days,
|
||||
"GC_LOCK_FILE_MAX_AGE_HOURS": self.gc_lock_file_max_age_hours,
|
||||
"GC_DRY_RUN": self.gc_dry_run,
|
||||
"GC_IO_THROTTLE_MS": self.gc_io_throttle_ms,
|
||||
"INTEGRITY_ENABLED": self.integrity_enabled,
|
||||
"INTEGRITY_INTERVAL_HOURS": self.integrity_interval_hours,
|
||||
"INTEGRITY_BATCH_SIZE": self.integrity_batch_size,
|
||||
"INTEGRITY_AUTO_HEAL": self.integrity_auto_heal,
|
||||
"INTEGRITY_DRY_RUN": self.integrity_dry_run,
|
||||
"INTEGRITY_IO_THROTTLE_MS": self.integrity_io_throttle_ms,
|
||||
}
|
||||
|
||||
@@ -190,6 +190,15 @@ class EncryptedObjectStorage:
|
||||
def list_objects(self, bucket_name: str, **kwargs):
|
||||
return self.storage.list_objects(bucket_name, **kwargs)
|
||||
|
||||
def list_objects_shallow(self, bucket_name: str, **kwargs):
|
||||
return self.storage.list_objects_shallow(bucket_name, **kwargs)
|
||||
|
||||
def iter_objects_shallow(self, bucket_name: str, **kwargs):
|
||||
return self.storage.iter_objects_shallow(bucket_name, **kwargs)
|
||||
|
||||
def search_objects(self, bucket_name: str, query: str, **kwargs):
|
||||
return self.storage.search_objects(bucket_name, query, **kwargs)
|
||||
|
||||
def list_objects_all(self, bucket_name: str):
|
||||
return self.storage.list_objects_all(bucket_name)
|
||||
|
||||
@@ -274,5 +283,11 @@ class EncryptedObjectStorage:
|
||||
def set_bucket_quota(self, bucket_name: str, *, max_bytes=None, max_objects=None):
|
||||
return self.storage.set_bucket_quota(bucket_name, max_bytes=max_bytes, max_objects=max_objects)
|
||||
|
||||
def get_bucket_website(self, bucket_name: str):
|
||||
return self.storage.get_bucket_website(bucket_name)
|
||||
|
||||
def set_bucket_website(self, bucket_name: str, website_config):
|
||||
return self.storage.set_bucket_website(bucket_name, website_config)
|
||||
|
||||
def _compute_etag(self, path: Path) -> str:
|
||||
return self.storage._compute_etag(path)
|
||||
|
||||
@@ -19,6 +19,13 @@ from cryptography.hazmat.primitives import hashes
|
||||
if sys.platform != "win32":
|
||||
import fcntl
|
||||
|
||||
try:
|
||||
import myfsio_core as _rc
|
||||
_HAS_RUST = True
|
||||
except ImportError:
|
||||
_rc = None
|
||||
_HAS_RUST = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -338,6 +345,69 @@ class StreamingEncryptor:
|
||||
output.seek(0)
|
||||
return output
|
||||
|
||||
def encrypt_file(self, input_path: str, output_path: str) -> EncryptionMetadata:
|
||||
data_key, encrypted_data_key = self.provider.generate_data_key()
|
||||
base_nonce = secrets.token_bytes(12)
|
||||
|
||||
if _HAS_RUST:
|
||||
_rc.encrypt_stream_chunked(
|
||||
input_path, output_path, data_key, base_nonce, self.chunk_size
|
||||
)
|
||||
else:
|
||||
with open(input_path, "rb") as stream:
|
||||
aesgcm = AESGCM(data_key)
|
||||
with open(output_path, "wb") as out:
|
||||
out.write(b"\x00\x00\x00\x00")
|
||||
chunk_index = 0
|
||||
while True:
|
||||
chunk = stream.read(self.chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
chunk_nonce = self._derive_chunk_nonce(base_nonce, chunk_index)
|
||||
encrypted_chunk = aesgcm.encrypt(chunk_nonce, chunk, None)
|
||||
out.write(len(encrypted_chunk).to_bytes(self.HEADER_SIZE, "big"))
|
||||
out.write(encrypted_chunk)
|
||||
chunk_index += 1
|
||||
out.seek(0)
|
||||
out.write(chunk_index.to_bytes(4, "big"))
|
||||
|
||||
return EncryptionMetadata(
|
||||
algorithm="AES256",
|
||||
key_id=self.provider.KEY_ID if hasattr(self.provider, "KEY_ID") else "local",
|
||||
nonce=base_nonce,
|
||||
encrypted_data_key=encrypted_data_key,
|
||||
)
|
||||
|
||||
def decrypt_file(self, input_path: str, output_path: str,
|
||||
metadata: EncryptionMetadata) -> None:
|
||||
data_key = self.provider.decrypt_data_key(metadata.encrypted_data_key, metadata.key_id)
|
||||
base_nonce = metadata.nonce
|
||||
|
||||
if _HAS_RUST:
|
||||
_rc.decrypt_stream_chunked(input_path, output_path, data_key, base_nonce)
|
||||
else:
|
||||
with open(input_path, "rb") as stream:
|
||||
chunk_count_bytes = stream.read(4)
|
||||
if len(chunk_count_bytes) < 4:
|
||||
raise EncryptionError("Invalid encrypted stream: missing header")
|
||||
chunk_count = int.from_bytes(chunk_count_bytes, "big")
|
||||
aesgcm = AESGCM(data_key)
|
||||
with open(output_path, "wb") as out:
|
||||
for chunk_index in range(chunk_count):
|
||||
size_bytes = stream.read(self.HEADER_SIZE)
|
||||
if len(size_bytes) < self.HEADER_SIZE:
|
||||
raise EncryptionError(f"Invalid encrypted stream: truncated at chunk {chunk_index}")
|
||||
chunk_size = int.from_bytes(size_bytes, "big")
|
||||
encrypted_chunk = stream.read(chunk_size)
|
||||
if len(encrypted_chunk) < chunk_size:
|
||||
raise EncryptionError(f"Invalid encrypted stream: incomplete chunk {chunk_index}")
|
||||
chunk_nonce = self._derive_chunk_nonce(base_nonce, chunk_index)
|
||||
try:
|
||||
decrypted_chunk = aesgcm.decrypt(chunk_nonce, encrypted_chunk, None)
|
||||
out.write(decrypted_chunk)
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"Failed to decrypt chunk {chunk_index}: {exc}") from exc
|
||||
|
||||
|
||||
class EncryptionManager:
|
||||
"""Manages encryption providers and operations."""
|
||||
|
||||
@@ -175,13 +175,21 @@ def handle_app_error(error: AppError) -> Response:
|
||||
|
||||
def handle_rate_limit_exceeded(e: RateLimitExceeded) -> Response:
|
||||
g.s3_error_code = "SlowDown"
|
||||
if request.path.startswith("/ui") or request.path.startswith("/buckets"):
|
||||
wants_json = (
|
||||
request.is_json or
|
||||
request.headers.get("X-Requested-With") == "XMLHttpRequest" or
|
||||
"application/json" in request.accept_mimetypes.values()
|
||||
)
|
||||
if wants_json:
|
||||
return jsonify({"success": False, "error": {"code": "SlowDown", "message": "Please reduce your request rate."}}), 429
|
||||
error = Element("Error")
|
||||
SubElement(error, "Code").text = "SlowDown"
|
||||
SubElement(error, "Message").text = "Please reduce your request rate."
|
||||
SubElement(error, "Resource").text = request.path
|
||||
SubElement(error, "RequestId").text = getattr(g, "request_id", "")
|
||||
xml_bytes = tostring(error, encoding="utf-8")
|
||||
return Response(xml_bytes, status=429, mimetype="application/xml")
|
||||
return Response(xml_bytes, status="429 Too Many Requests", mimetype="application/xml")
|
||||
|
||||
|
||||
def register_error_handlers(app):
|
||||
|
||||
596
app/gc.py
Normal file
596
app/gc.py
Normal file
@@ -0,0 +1,596 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GCResult:
|
||||
temp_files_deleted: int = 0
|
||||
temp_bytes_freed: int = 0
|
||||
multipart_uploads_deleted: int = 0
|
||||
multipart_bytes_freed: int = 0
|
||||
lock_files_deleted: int = 0
|
||||
orphaned_metadata_deleted: int = 0
|
||||
orphaned_versions_deleted: int = 0
|
||||
orphaned_version_bytes_freed: int = 0
|
||||
empty_dirs_removed: int = 0
|
||||
errors: List[str] = field(default_factory=list)
|
||||
execution_time_seconds: float = 0.0
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"temp_files_deleted": self.temp_files_deleted,
|
||||
"temp_bytes_freed": self.temp_bytes_freed,
|
||||
"multipart_uploads_deleted": self.multipart_uploads_deleted,
|
||||
"multipart_bytes_freed": self.multipart_bytes_freed,
|
||||
"lock_files_deleted": self.lock_files_deleted,
|
||||
"orphaned_metadata_deleted": self.orphaned_metadata_deleted,
|
||||
"orphaned_versions_deleted": self.orphaned_versions_deleted,
|
||||
"orphaned_version_bytes_freed": self.orphaned_version_bytes_freed,
|
||||
"empty_dirs_removed": self.empty_dirs_removed,
|
||||
"errors": self.errors,
|
||||
"execution_time_seconds": self.execution_time_seconds,
|
||||
}
|
||||
|
||||
@property
|
||||
def total_bytes_freed(self) -> int:
|
||||
return self.temp_bytes_freed + self.multipart_bytes_freed + self.orphaned_version_bytes_freed
|
||||
|
||||
@property
|
||||
def has_work(self) -> bool:
|
||||
return (
|
||||
self.temp_files_deleted > 0
|
||||
or self.multipart_uploads_deleted > 0
|
||||
or self.lock_files_deleted > 0
|
||||
or self.orphaned_metadata_deleted > 0
|
||||
or self.orphaned_versions_deleted > 0
|
||||
or self.empty_dirs_removed > 0
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GCExecutionRecord:
|
||||
timestamp: float
|
||||
result: dict
|
||||
dry_run: bool
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"timestamp": self.timestamp,
|
||||
"result": self.result,
|
||||
"dry_run": self.dry_run,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> GCExecutionRecord:
|
||||
return cls(
|
||||
timestamp=data["timestamp"],
|
||||
result=data["result"],
|
||||
dry_run=data.get("dry_run", False),
|
||||
)
|
||||
|
||||
|
||||
class GCHistoryStore:
|
||||
def __init__(self, storage_root: Path, max_records: int = 50) -> None:
|
||||
self.storage_root = storage_root
|
||||
self.max_records = max_records
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _get_path(self) -> Path:
|
||||
return self.storage_root / ".myfsio.sys" / "config" / "gc_history.json"
|
||||
|
||||
def load(self) -> List[GCExecutionRecord]:
|
||||
path = self._get_path()
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
return [GCExecutionRecord.from_dict(d) for d in data.get("executions", [])]
|
||||
except (OSError, ValueError, KeyError) as e:
|
||||
logger.error("Failed to load GC history: %s", e)
|
||||
return []
|
||||
|
||||
def save(self, records: List[GCExecutionRecord]) -> None:
|
||||
path = self._get_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {"executions": [r.to_dict() for r in records[: self.max_records]]}
|
||||
try:
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
except OSError as e:
|
||||
logger.error("Failed to save GC history: %s", e)
|
||||
|
||||
def add(self, record: GCExecutionRecord) -> None:
|
||||
with self._lock:
|
||||
records = self.load()
|
||||
records.insert(0, record)
|
||||
self.save(records)
|
||||
|
||||
def get_history(self, limit: int = 50, offset: int = 0) -> List[GCExecutionRecord]:
|
||||
return self.load()[offset : offset + limit]
|
||||
|
||||
|
||||
def _dir_size(path: Path) -> int:
|
||||
total = 0
|
||||
try:
|
||||
for f in path.rglob("*"):
|
||||
if f.is_file():
|
||||
try:
|
||||
total += f.stat().st_size
|
||||
except OSError:
|
||||
pass
|
||||
except OSError:
|
||||
pass
|
||||
return total
|
||||
|
||||
|
||||
def _file_age_hours(path: Path) -> float:
|
||||
try:
|
||||
mtime = path.stat().st_mtime
|
||||
return (time.time() - mtime) / 3600.0
|
||||
except OSError:
|
||||
return 0.0
|
||||
|
||||
|
||||
class GarbageCollector:
|
||||
SYSTEM_ROOT = ".myfsio.sys"
|
||||
SYSTEM_TMP_DIR = "tmp"
|
||||
SYSTEM_MULTIPART_DIR = "multipart"
|
||||
SYSTEM_BUCKETS_DIR = "buckets"
|
||||
BUCKET_META_DIR = "meta"
|
||||
BUCKET_VERSIONS_DIR = "versions"
|
||||
INTERNAL_FOLDERS = {".meta", ".versions", ".multipart"}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
storage_root: Path,
|
||||
interval_hours: float = 6.0,
|
||||
temp_file_max_age_hours: float = 24.0,
|
||||
multipart_max_age_days: int = 7,
|
||||
lock_file_max_age_hours: float = 1.0,
|
||||
dry_run: bool = False,
|
||||
max_history: int = 50,
|
||||
io_throttle_ms: int = 10,
|
||||
) -> None:
|
||||
self.storage_root = Path(storage_root)
|
||||
self.interval_seconds = interval_hours * 3600.0
|
||||
self.temp_file_max_age_hours = temp_file_max_age_hours
|
||||
self.multipart_max_age_days = multipart_max_age_days
|
||||
self.lock_file_max_age_hours = lock_file_max_age_hours
|
||||
self.dry_run = dry_run
|
||||
self._timer: Optional[threading.Timer] = None
|
||||
self._shutdown = False
|
||||
self._lock = threading.Lock()
|
||||
self._scanning = False
|
||||
self._scan_start_time: Optional[float] = None
|
||||
self._io_throttle = max(0, io_throttle_ms) / 1000.0
|
||||
self.history_store = GCHistoryStore(storage_root, max_records=max_history)
|
||||
|
||||
def start(self) -> None:
|
||||
if self._timer is not None:
|
||||
return
|
||||
self._shutdown = False
|
||||
self._schedule_next()
|
||||
logger.info(
|
||||
"GC started: interval=%.1fh, temp_max_age=%.1fh, multipart_max_age=%dd, lock_max_age=%.1fh, dry_run=%s",
|
||||
self.interval_seconds / 3600.0,
|
||||
self.temp_file_max_age_hours,
|
||||
self.multipart_max_age_days,
|
||||
self.lock_file_max_age_hours,
|
||||
self.dry_run,
|
||||
)
|
||||
|
||||
def stop(self) -> None:
|
||||
self._shutdown = True
|
||||
if self._timer:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
logger.info("GC stopped")
|
||||
|
||||
def _schedule_next(self) -> None:
|
||||
if self._shutdown:
|
||||
return
|
||||
self._timer = threading.Timer(self.interval_seconds, self._run_cycle)
|
||||
self._timer.daemon = True
|
||||
self._timer.start()
|
||||
|
||||
def _run_cycle(self) -> None:
|
||||
if self._shutdown:
|
||||
return
|
||||
try:
|
||||
self.run_now()
|
||||
except Exception as e:
|
||||
logger.error("GC cycle failed: %s", e)
|
||||
finally:
|
||||
self._schedule_next()
|
||||
|
||||
def run_now(self, dry_run: Optional[bool] = None) -> GCResult:
|
||||
if not self._lock.acquire(blocking=False):
|
||||
raise RuntimeError("GC is already in progress")
|
||||
|
||||
effective_dry_run = dry_run if dry_run is not None else self.dry_run
|
||||
|
||||
try:
|
||||
self._scanning = True
|
||||
self._scan_start_time = time.time()
|
||||
|
||||
start = self._scan_start_time
|
||||
result = GCResult()
|
||||
|
||||
original_dry_run = self.dry_run
|
||||
self.dry_run = effective_dry_run
|
||||
try:
|
||||
self._clean_temp_files(result)
|
||||
self._clean_orphaned_multipart(result)
|
||||
self._clean_stale_locks(result)
|
||||
self._clean_orphaned_metadata(result)
|
||||
self._clean_orphaned_versions(result)
|
||||
self._clean_empty_dirs(result)
|
||||
finally:
|
||||
self.dry_run = original_dry_run
|
||||
|
||||
result.execution_time_seconds = time.time() - start
|
||||
|
||||
if result.has_work or result.errors:
|
||||
logger.info(
|
||||
"GC completed in %.2fs: temp=%d (%.1f MB), multipart=%d (%.1f MB), "
|
||||
"locks=%d, meta=%d, versions=%d (%.1f MB), dirs=%d, errors=%d%s",
|
||||
result.execution_time_seconds,
|
||||
result.temp_files_deleted,
|
||||
result.temp_bytes_freed / (1024 * 1024),
|
||||
result.multipart_uploads_deleted,
|
||||
result.multipart_bytes_freed / (1024 * 1024),
|
||||
result.lock_files_deleted,
|
||||
result.orphaned_metadata_deleted,
|
||||
result.orphaned_versions_deleted,
|
||||
result.orphaned_version_bytes_freed / (1024 * 1024),
|
||||
result.empty_dirs_removed,
|
||||
len(result.errors),
|
||||
" (dry run)" if effective_dry_run else "",
|
||||
)
|
||||
|
||||
record = GCExecutionRecord(
|
||||
timestamp=time.time(),
|
||||
result=result.to_dict(),
|
||||
dry_run=effective_dry_run,
|
||||
)
|
||||
self.history_store.add(record)
|
||||
|
||||
return result
|
||||
finally:
|
||||
self._scanning = False
|
||||
self._scan_start_time = None
|
||||
self._lock.release()
|
||||
|
||||
def run_async(self, dry_run: Optional[bool] = None) -> bool:
|
||||
if self._scanning:
|
||||
return False
|
||||
t = threading.Thread(target=self.run_now, args=(dry_run,), daemon=True)
|
||||
t.start()
|
||||
return True
|
||||
|
||||
def _system_path(self) -> Path:
|
||||
return self.storage_root / self.SYSTEM_ROOT
|
||||
|
||||
def _throttle(self) -> bool:
|
||||
if self._shutdown:
|
||||
return True
|
||||
if self._io_throttle > 0:
|
||||
time.sleep(self._io_throttle)
|
||||
return self._shutdown
|
||||
|
||||
def _list_bucket_names(self) -> List[str]:
|
||||
names = []
|
||||
try:
|
||||
for entry in self.storage_root.iterdir():
|
||||
if entry.is_dir() and entry.name != self.SYSTEM_ROOT:
|
||||
names.append(entry.name)
|
||||
except OSError:
|
||||
pass
|
||||
return names
|
||||
|
||||
def _clean_temp_files(self, result: GCResult) -> None:
|
||||
tmp_dir = self._system_path() / self.SYSTEM_TMP_DIR
|
||||
if not tmp_dir.exists():
|
||||
return
|
||||
try:
|
||||
for entry in tmp_dir.iterdir():
|
||||
if self._throttle():
|
||||
return
|
||||
if not entry.is_file():
|
||||
continue
|
||||
age = _file_age_hours(entry)
|
||||
if age < self.temp_file_max_age_hours:
|
||||
continue
|
||||
try:
|
||||
size = entry.stat().st_size
|
||||
if not self.dry_run:
|
||||
entry.unlink()
|
||||
result.temp_files_deleted += 1
|
||||
result.temp_bytes_freed += size
|
||||
except OSError as e:
|
||||
result.errors.append(f"temp file {entry.name}: {e}")
|
||||
except OSError as e:
|
||||
result.errors.append(f"scan tmp dir: {e}")
|
||||
|
||||
def _clean_orphaned_multipart(self, result: GCResult) -> None:
|
||||
cutoff_hours = self.multipart_max_age_days * 24.0
|
||||
bucket_names = self._list_bucket_names()
|
||||
|
||||
for bucket_name in bucket_names:
|
||||
if self._shutdown:
|
||||
return
|
||||
for multipart_root in (
|
||||
self._system_path() / self.SYSTEM_MULTIPART_DIR / bucket_name,
|
||||
self.storage_root / bucket_name / ".multipart",
|
||||
):
|
||||
if not multipart_root.exists():
|
||||
continue
|
||||
try:
|
||||
for upload_dir in multipart_root.iterdir():
|
||||
if self._throttle():
|
||||
return
|
||||
if not upload_dir.is_dir():
|
||||
continue
|
||||
self._maybe_clean_upload(upload_dir, cutoff_hours, result)
|
||||
except OSError as e:
|
||||
result.errors.append(f"scan multipart {bucket_name}: {e}")
|
||||
|
||||
def _maybe_clean_upload(self, upload_dir: Path, cutoff_hours: float, result: GCResult) -> None:
|
||||
manifest_path = upload_dir / "manifest.json"
|
||||
age = _file_age_hours(manifest_path) if manifest_path.exists() else _file_age_hours(upload_dir)
|
||||
|
||||
if age < cutoff_hours:
|
||||
return
|
||||
|
||||
dir_bytes = _dir_size(upload_dir)
|
||||
try:
|
||||
if not self.dry_run:
|
||||
shutil.rmtree(upload_dir, ignore_errors=True)
|
||||
result.multipart_uploads_deleted += 1
|
||||
result.multipart_bytes_freed += dir_bytes
|
||||
except OSError as e:
|
||||
result.errors.append(f"multipart {upload_dir.name}: {e}")
|
||||
|
||||
def _clean_stale_locks(self, result: GCResult) -> None:
|
||||
buckets_root = self._system_path() / self.SYSTEM_BUCKETS_DIR
|
||||
if not buckets_root.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
for bucket_dir in buckets_root.iterdir():
|
||||
if self._shutdown:
|
||||
return
|
||||
if not bucket_dir.is_dir():
|
||||
continue
|
||||
locks_dir = bucket_dir / "locks"
|
||||
if not locks_dir.exists():
|
||||
continue
|
||||
try:
|
||||
for lock_file in locks_dir.iterdir():
|
||||
if self._throttle():
|
||||
return
|
||||
if not lock_file.is_file() or not lock_file.name.endswith(".lock"):
|
||||
continue
|
||||
age = _file_age_hours(lock_file)
|
||||
if age < self.lock_file_max_age_hours:
|
||||
continue
|
||||
try:
|
||||
if not self.dry_run:
|
||||
lock_file.unlink(missing_ok=True)
|
||||
result.lock_files_deleted += 1
|
||||
except OSError as e:
|
||||
result.errors.append(f"lock {lock_file.name}: {e}")
|
||||
except OSError as e:
|
||||
result.errors.append(f"scan locks {bucket_dir.name}: {e}")
|
||||
except OSError as e:
|
||||
result.errors.append(f"scan buckets for locks: {e}")
|
||||
|
||||
def _clean_orphaned_metadata(self, result: GCResult) -> None:
|
||||
bucket_names = self._list_bucket_names()
|
||||
|
||||
for bucket_name in bucket_names:
|
||||
if self._shutdown:
|
||||
return
|
||||
legacy_meta = self.storage_root / bucket_name / ".meta"
|
||||
if legacy_meta.exists():
|
||||
self._clean_legacy_metadata(bucket_name, legacy_meta, result)
|
||||
|
||||
new_meta = self._system_path() / self.SYSTEM_BUCKETS_DIR / bucket_name / self.BUCKET_META_DIR
|
||||
if new_meta.exists():
|
||||
self._clean_index_metadata(bucket_name, new_meta, result)
|
||||
|
||||
def _clean_legacy_metadata(self, bucket_name: str, meta_root: Path, result: GCResult) -> None:
|
||||
bucket_path = self.storage_root / bucket_name
|
||||
try:
|
||||
for meta_file in meta_root.rglob("*.meta.json"):
|
||||
if self._throttle():
|
||||
return
|
||||
if not meta_file.is_file():
|
||||
continue
|
||||
try:
|
||||
rel = meta_file.relative_to(meta_root)
|
||||
object_key = rel.as_posix().removesuffix(".meta.json")
|
||||
object_path = bucket_path / object_key
|
||||
if not object_path.exists():
|
||||
if not self.dry_run:
|
||||
meta_file.unlink(missing_ok=True)
|
||||
result.orphaned_metadata_deleted += 1
|
||||
except (OSError, ValueError) as e:
|
||||
result.errors.append(f"legacy meta {bucket_name}/{meta_file.name}: {e}")
|
||||
except OSError as e:
|
||||
result.errors.append(f"scan legacy meta {bucket_name}: {e}")
|
||||
|
||||
def _clean_index_metadata(self, bucket_name: str, meta_root: Path, result: GCResult) -> None:
|
||||
bucket_path = self.storage_root / bucket_name
|
||||
try:
|
||||
for index_file in meta_root.rglob("_index.json"):
|
||||
if self._throttle():
|
||||
return
|
||||
if not index_file.is_file():
|
||||
continue
|
||||
try:
|
||||
with open(index_file, "r", encoding="utf-8") as f:
|
||||
index_data = json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
|
||||
keys_to_remove = []
|
||||
for key in index_data:
|
||||
rel_dir = index_file.parent.relative_to(meta_root)
|
||||
if rel_dir == Path("."):
|
||||
full_key = key
|
||||
else:
|
||||
full_key = rel_dir.as_posix() + "/" + key
|
||||
object_path = bucket_path / full_key
|
||||
if not object_path.exists():
|
||||
keys_to_remove.append(key)
|
||||
|
||||
if keys_to_remove:
|
||||
if not self.dry_run:
|
||||
for k in keys_to_remove:
|
||||
index_data.pop(k, None)
|
||||
if index_data:
|
||||
try:
|
||||
with open(index_file, "w", encoding="utf-8") as f:
|
||||
json.dump(index_data, f)
|
||||
except OSError as e:
|
||||
result.errors.append(f"write index {bucket_name}: {e}")
|
||||
continue
|
||||
else:
|
||||
try:
|
||||
index_file.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
result.orphaned_metadata_deleted += len(keys_to_remove)
|
||||
except OSError as e:
|
||||
result.errors.append(f"scan index meta {bucket_name}: {e}")
|
||||
|
||||
def _clean_orphaned_versions(self, result: GCResult) -> None:
|
||||
bucket_names = self._list_bucket_names()
|
||||
|
||||
for bucket_name in bucket_names:
|
||||
if self._shutdown:
|
||||
return
|
||||
bucket_path = self.storage_root / bucket_name
|
||||
for versions_root in (
|
||||
self._system_path() / self.SYSTEM_BUCKETS_DIR / bucket_name / self.BUCKET_VERSIONS_DIR,
|
||||
self.storage_root / bucket_name / ".versions",
|
||||
):
|
||||
if not versions_root.exists():
|
||||
continue
|
||||
try:
|
||||
for key_dir in versions_root.iterdir():
|
||||
if self._throttle():
|
||||
return
|
||||
if not key_dir.is_dir():
|
||||
continue
|
||||
self._clean_versions_for_key(bucket_path, versions_root, key_dir, result)
|
||||
except OSError as e:
|
||||
result.errors.append(f"scan versions {bucket_name}: {e}")
|
||||
|
||||
def _clean_versions_for_key(
|
||||
self, bucket_path: Path, versions_root: Path, key_dir: Path, result: GCResult
|
||||
) -> None:
|
||||
try:
|
||||
rel = key_dir.relative_to(versions_root)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
object_path = bucket_path / rel
|
||||
if object_path.exists():
|
||||
return
|
||||
|
||||
version_files = list(key_dir.glob("*.bin")) + list(key_dir.glob("*.json"))
|
||||
if not version_files:
|
||||
return
|
||||
|
||||
for vf in version_files:
|
||||
try:
|
||||
size = vf.stat().st_size if vf.suffix == ".bin" else 0
|
||||
if not self.dry_run:
|
||||
vf.unlink(missing_ok=True)
|
||||
if vf.suffix == ".bin":
|
||||
result.orphaned_version_bytes_freed += size
|
||||
result.orphaned_versions_deleted += 1
|
||||
except OSError as e:
|
||||
result.errors.append(f"version file {vf.name}: {e}")
|
||||
|
||||
def _clean_empty_dirs(self, result: GCResult) -> None:
|
||||
targets = [
|
||||
self._system_path() / self.SYSTEM_TMP_DIR,
|
||||
self._system_path() / self.SYSTEM_MULTIPART_DIR,
|
||||
self._system_path() / self.SYSTEM_BUCKETS_DIR,
|
||||
]
|
||||
for bucket_name in self._list_bucket_names():
|
||||
targets.append(self.storage_root / bucket_name / ".meta")
|
||||
targets.append(self.storage_root / bucket_name / ".versions")
|
||||
targets.append(self.storage_root / bucket_name / ".multipart")
|
||||
|
||||
for root in targets:
|
||||
if not root.exists():
|
||||
continue
|
||||
self._remove_empty_dirs_recursive(root, root, result)
|
||||
|
||||
def _remove_empty_dirs_recursive(self, path: Path, stop_at: Path, result: GCResult) -> bool:
|
||||
if self._shutdown:
|
||||
return False
|
||||
if not path.is_dir():
|
||||
return False
|
||||
|
||||
try:
|
||||
children = list(path.iterdir())
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
all_empty = True
|
||||
for child in children:
|
||||
if self._throttle():
|
||||
return False
|
||||
if child.is_dir():
|
||||
if not self._remove_empty_dirs_recursive(child, stop_at, result):
|
||||
all_empty = False
|
||||
else:
|
||||
all_empty = False
|
||||
|
||||
if all_empty and path != stop_at:
|
||||
try:
|
||||
if not self.dry_run:
|
||||
path.rmdir()
|
||||
result.empty_dirs_removed += 1
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
return all_empty
|
||||
|
||||
def get_history(self, limit: int = 50, offset: int = 0) -> List[dict]:
|
||||
records = self.history_store.get_history(limit, offset)
|
||||
return [r.to_dict() for r in records]
|
||||
|
||||
def get_status(self) -> dict:
|
||||
status: Dict[str, Any] = {
|
||||
"enabled": not self._shutdown or self._timer is not None,
|
||||
"running": self._timer is not None and not self._shutdown,
|
||||
"scanning": self._scanning,
|
||||
"interval_hours": self.interval_seconds / 3600.0,
|
||||
"temp_file_max_age_hours": self.temp_file_max_age_hours,
|
||||
"multipart_max_age_days": self.multipart_max_age_days,
|
||||
"lock_file_max_age_hours": self.lock_file_max_age_hours,
|
||||
"dry_run": self.dry_run,
|
||||
"io_throttle_ms": round(self._io_throttle * 1000),
|
||||
}
|
||||
if self._scanning and self._scan_start_time:
|
||||
status["scan_elapsed_seconds"] = time.time() - self._scan_start_time
|
||||
return status
|
||||
650
app/iam.py
650
app/iam.py
File diff suppressed because it is too large
Load Diff
882
app/integrity.py
Normal file
882
app/integrity.py
Normal file
@@ -0,0 +1,882 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
try:
|
||||
import myfsio_core as _rc
|
||||
_HAS_RUST = True
|
||||
except ImportError:
|
||||
_HAS_RUST = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _compute_etag(path: Path) -> str:
|
||||
if _HAS_RUST:
|
||||
return _rc.md5_file(str(path))
|
||||
checksum = hashlib.md5()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(8192), b""):
|
||||
checksum.update(chunk)
|
||||
return checksum.hexdigest()
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntegrityIssue:
|
||||
issue_type: str
|
||||
bucket: str
|
||||
key: str
|
||||
detail: str
|
||||
healed: bool = False
|
||||
heal_action: str = ""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"issue_type": self.issue_type,
|
||||
"bucket": self.bucket,
|
||||
"key": self.key,
|
||||
"detail": self.detail,
|
||||
"healed": self.healed,
|
||||
"heal_action": self.heal_action,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntegrityResult:
|
||||
corrupted_objects: int = 0
|
||||
orphaned_objects: int = 0
|
||||
phantom_metadata: int = 0
|
||||
stale_versions: int = 0
|
||||
etag_cache_inconsistencies: int = 0
|
||||
legacy_metadata_drifts: int = 0
|
||||
issues_healed: int = 0
|
||||
issues: List[IntegrityIssue] = field(default_factory=list)
|
||||
errors: List[str] = field(default_factory=list)
|
||||
objects_scanned: int = 0
|
||||
buckets_scanned: int = 0
|
||||
execution_time_seconds: float = 0.0
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"corrupted_objects": self.corrupted_objects,
|
||||
"orphaned_objects": self.orphaned_objects,
|
||||
"phantom_metadata": self.phantom_metadata,
|
||||
"stale_versions": self.stale_versions,
|
||||
"etag_cache_inconsistencies": self.etag_cache_inconsistencies,
|
||||
"legacy_metadata_drifts": self.legacy_metadata_drifts,
|
||||
"issues_healed": self.issues_healed,
|
||||
"issues": [i.to_dict() for i in self.issues],
|
||||
"errors": self.errors,
|
||||
"objects_scanned": self.objects_scanned,
|
||||
"buckets_scanned": self.buckets_scanned,
|
||||
"execution_time_seconds": self.execution_time_seconds,
|
||||
}
|
||||
|
||||
@property
|
||||
def total_issues(self) -> int:
|
||||
return (
|
||||
self.corrupted_objects
|
||||
+ self.orphaned_objects
|
||||
+ self.phantom_metadata
|
||||
+ self.stale_versions
|
||||
+ self.etag_cache_inconsistencies
|
||||
+ self.legacy_metadata_drifts
|
||||
)
|
||||
|
||||
@property
|
||||
def has_issues(self) -> bool:
|
||||
return self.total_issues > 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntegrityExecutionRecord:
|
||||
timestamp: float
|
||||
result: dict
|
||||
dry_run: bool
|
||||
auto_heal: bool
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"timestamp": self.timestamp,
|
||||
"result": self.result,
|
||||
"dry_run": self.dry_run,
|
||||
"auto_heal": self.auto_heal,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> IntegrityExecutionRecord:
|
||||
return cls(
|
||||
timestamp=data["timestamp"],
|
||||
result=data["result"],
|
||||
dry_run=data.get("dry_run", False),
|
||||
auto_heal=data.get("auto_heal", False),
|
||||
)
|
||||
|
||||
|
||||
class IntegrityHistoryStore:
|
||||
def __init__(self, storage_root: Path, max_records: int = 50) -> None:
|
||||
self.storage_root = storage_root
|
||||
self.max_records = max_records
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _get_path(self) -> Path:
|
||||
return self.storage_root / ".myfsio.sys" / "config" / "integrity_history.json"
|
||||
|
||||
def load(self) -> List[IntegrityExecutionRecord]:
|
||||
path = self._get_path()
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
return [IntegrityExecutionRecord.from_dict(d) for d in data.get("executions", [])]
|
||||
except (OSError, ValueError, KeyError) as e:
|
||||
logger.error("Failed to load integrity history: %s", e)
|
||||
return []
|
||||
|
||||
def save(self, records: List[IntegrityExecutionRecord]) -> None:
|
||||
path = self._get_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {"executions": [r.to_dict() for r in records[: self.max_records]]}
|
||||
try:
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
except OSError as e:
|
||||
logger.error("Failed to save integrity history: %s", e)
|
||||
|
||||
def add(self, record: IntegrityExecutionRecord) -> None:
|
||||
with self._lock:
|
||||
records = self.load()
|
||||
records.insert(0, record)
|
||||
self.save(records)
|
||||
|
||||
def get_history(self, limit: int = 50, offset: int = 0) -> List[IntegrityExecutionRecord]:
|
||||
return self.load()[offset : offset + limit]
|
||||
|
||||
|
||||
class IntegrityCursorStore:
|
||||
def __init__(self, storage_root: Path) -> None:
|
||||
self.storage_root = storage_root
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _get_path(self) -> Path:
|
||||
return self.storage_root / ".myfsio.sys" / "config" / "integrity_cursor.json"
|
||||
|
||||
def load(self) -> Dict[str, Any]:
|
||||
path = self._get_path()
|
||||
if not path.exists():
|
||||
return {"buckets": {}}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data.get("buckets"), dict):
|
||||
return {"buckets": {}}
|
||||
return data
|
||||
except (OSError, ValueError, KeyError):
|
||||
return {"buckets": {}}
|
||||
|
||||
def save(self, data: Dict[str, Any]) -> None:
|
||||
path = self._get_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
except OSError as e:
|
||||
logger.error("Failed to save integrity cursor: %s", e)
|
||||
|
||||
def update_bucket(self, bucket_name: str, timestamp: float) -> None:
|
||||
with self._lock:
|
||||
data = self.load()
|
||||
data["buckets"][bucket_name] = {"last_scanned": timestamp}
|
||||
self.save(data)
|
||||
|
||||
def clean_stale(self, existing_buckets: List[str]) -> None:
|
||||
with self._lock:
|
||||
data = self.load()
|
||||
existing_set = set(existing_buckets)
|
||||
stale_keys = [k for k in data["buckets"] if k not in existing_set]
|
||||
if stale_keys:
|
||||
for k in stale_keys:
|
||||
del data["buckets"][k]
|
||||
self.save(data)
|
||||
|
||||
def get_bucket_order(self, bucket_names: List[str]) -> List[str]:
|
||||
data = self.load()
|
||||
buckets_info = data.get("buckets", {})
|
||||
|
||||
def sort_key(name: str) -> float:
|
||||
entry = buckets_info.get(name)
|
||||
if entry is None:
|
||||
return 0.0
|
||||
return entry.get("last_scanned", 0.0)
|
||||
|
||||
return sorted(bucket_names, key=sort_key)
|
||||
|
||||
def get_info(self) -> Dict[str, Any]:
|
||||
data = self.load()
|
||||
buckets = data.get("buckets", {})
|
||||
return {
|
||||
"tracked_buckets": len(buckets),
|
||||
"buckets": {
|
||||
name: info.get("last_scanned")
|
||||
for name, info in buckets.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
MAX_ISSUES = 500
|
||||
|
||||
|
||||
class IntegrityChecker:
|
||||
SYSTEM_ROOT = ".myfsio.sys"
|
||||
SYSTEM_BUCKETS_DIR = "buckets"
|
||||
BUCKET_META_DIR = "meta"
|
||||
BUCKET_VERSIONS_DIR = "versions"
|
||||
INTERNAL_FOLDERS = {".meta", ".versions", ".multipart"}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
storage_root: Path,
|
||||
interval_hours: float = 24.0,
|
||||
batch_size: int = 1000,
|
||||
auto_heal: bool = False,
|
||||
dry_run: bool = False,
|
||||
max_history: int = 50,
|
||||
io_throttle_ms: int = 10,
|
||||
) -> None:
|
||||
self.storage_root = Path(storage_root)
|
||||
self.interval_seconds = interval_hours * 3600.0
|
||||
self.batch_size = batch_size
|
||||
self.auto_heal = auto_heal
|
||||
self.dry_run = dry_run
|
||||
self._timer: Optional[threading.Timer] = None
|
||||
self._shutdown = False
|
||||
self._lock = threading.Lock()
|
||||
self._scanning = False
|
||||
self._scan_start_time: Optional[float] = None
|
||||
self._io_throttle = max(0, io_throttle_ms) / 1000.0
|
||||
self.history_store = IntegrityHistoryStore(storage_root, max_records=max_history)
|
||||
self.cursor_store = IntegrityCursorStore(self.storage_root)
|
||||
|
||||
def start(self) -> None:
|
||||
if self._timer is not None:
|
||||
return
|
||||
self._shutdown = False
|
||||
self._schedule_next()
|
||||
logger.info(
|
||||
"Integrity checker started: interval=%.1fh, batch_size=%d, auto_heal=%s, dry_run=%s",
|
||||
self.interval_seconds / 3600.0,
|
||||
self.batch_size,
|
||||
self.auto_heal,
|
||||
self.dry_run,
|
||||
)
|
||||
|
||||
def stop(self) -> None:
|
||||
self._shutdown = True
|
||||
if self._timer:
|
||||
self._timer.cancel()
|
||||
self._timer = None
|
||||
logger.info("Integrity checker stopped")
|
||||
|
||||
def _schedule_next(self) -> None:
|
||||
if self._shutdown:
|
||||
return
|
||||
self._timer = threading.Timer(self.interval_seconds, self._run_cycle)
|
||||
self._timer.daemon = True
|
||||
self._timer.start()
|
||||
|
||||
def _run_cycle(self) -> None:
|
||||
if self._shutdown:
|
||||
return
|
||||
try:
|
||||
self.run_now()
|
||||
except Exception as e:
|
||||
logger.error("Integrity check cycle failed: %s", e)
|
||||
finally:
|
||||
self._schedule_next()
|
||||
|
||||
def run_now(self, auto_heal: Optional[bool] = None, dry_run: Optional[bool] = None) -> IntegrityResult:
|
||||
if not self._lock.acquire(blocking=False):
|
||||
raise RuntimeError("Integrity scan is already in progress")
|
||||
|
||||
try:
|
||||
self._scanning = True
|
||||
self._scan_start_time = time.time()
|
||||
|
||||
effective_auto_heal = auto_heal if auto_heal is not None else self.auto_heal
|
||||
effective_dry_run = dry_run if dry_run is not None else self.dry_run
|
||||
|
||||
start = self._scan_start_time
|
||||
result = IntegrityResult()
|
||||
|
||||
bucket_names = self._list_bucket_names()
|
||||
self.cursor_store.clean_stale(bucket_names)
|
||||
ordered_buckets = self.cursor_store.get_bucket_order(bucket_names)
|
||||
|
||||
for bucket_name in ordered_buckets:
|
||||
if self._batch_exhausted(result):
|
||||
break
|
||||
result.buckets_scanned += 1
|
||||
self._check_corrupted_objects(bucket_name, result, effective_auto_heal, effective_dry_run)
|
||||
self._check_orphaned_objects(bucket_name, result, effective_auto_heal, effective_dry_run)
|
||||
self._check_phantom_metadata(bucket_name, result, effective_auto_heal, effective_dry_run)
|
||||
self._check_stale_versions(bucket_name, result, effective_auto_heal, effective_dry_run)
|
||||
self._check_etag_cache(bucket_name, result, effective_auto_heal, effective_dry_run)
|
||||
self._check_legacy_metadata(bucket_name, result, effective_auto_heal, effective_dry_run)
|
||||
self.cursor_store.update_bucket(bucket_name, time.time())
|
||||
|
||||
result.execution_time_seconds = time.time() - start
|
||||
|
||||
if result.has_issues or result.errors:
|
||||
logger.info(
|
||||
"Integrity check completed in %.2fs: corrupted=%d, orphaned=%d, phantom=%d, "
|
||||
"stale_versions=%d, etag_cache=%d, legacy_drift=%d, healed=%d, errors=%d%s",
|
||||
result.execution_time_seconds,
|
||||
result.corrupted_objects,
|
||||
result.orphaned_objects,
|
||||
result.phantom_metadata,
|
||||
result.stale_versions,
|
||||
result.etag_cache_inconsistencies,
|
||||
result.legacy_metadata_drifts,
|
||||
result.issues_healed,
|
||||
len(result.errors),
|
||||
" (dry run)" if effective_dry_run else "",
|
||||
)
|
||||
|
||||
record = IntegrityExecutionRecord(
|
||||
timestamp=time.time(),
|
||||
result=result.to_dict(),
|
||||
dry_run=effective_dry_run,
|
||||
auto_heal=effective_auto_heal,
|
||||
)
|
||||
self.history_store.add(record)
|
||||
|
||||
return result
|
||||
finally:
|
||||
self._scanning = False
|
||||
self._scan_start_time = None
|
||||
self._lock.release()
|
||||
|
||||
def run_async(self, auto_heal: Optional[bool] = None, dry_run: Optional[bool] = None) -> bool:
|
||||
if self._scanning:
|
||||
return False
|
||||
t = threading.Thread(target=self.run_now, args=(auto_heal, dry_run), daemon=True)
|
||||
t.start()
|
||||
return True
|
||||
|
||||
def _system_path(self) -> Path:
|
||||
return self.storage_root / self.SYSTEM_ROOT
|
||||
|
||||
def _list_bucket_names(self) -> List[str]:
|
||||
names = []
|
||||
try:
|
||||
for entry in self.storage_root.iterdir():
|
||||
if entry.is_dir() and entry.name != self.SYSTEM_ROOT:
|
||||
names.append(entry.name)
|
||||
except OSError:
|
||||
pass
|
||||
return names
|
||||
|
||||
def _throttle(self) -> bool:
|
||||
if self._shutdown:
|
||||
return True
|
||||
if self._io_throttle > 0:
|
||||
time.sleep(self._io_throttle)
|
||||
return self._shutdown
|
||||
|
||||
def _batch_exhausted(self, result: IntegrityResult) -> bool:
|
||||
return self._shutdown or result.objects_scanned >= self.batch_size
|
||||
|
||||
def _add_issue(self, result: IntegrityResult, issue: IntegrityIssue) -> None:
|
||||
if len(result.issues) < MAX_ISSUES:
|
||||
result.issues.append(issue)
|
||||
|
||||
def _check_corrupted_objects(
|
||||
self, bucket_name: str, result: IntegrityResult, auto_heal: bool, dry_run: bool
|
||||
) -> None:
|
||||
bucket_path = self.storage_root / bucket_name
|
||||
meta_root = self._system_path() / self.SYSTEM_BUCKETS_DIR / bucket_name / self.BUCKET_META_DIR
|
||||
|
||||
if not meta_root.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
for index_file in meta_root.rglob("_index.json"):
|
||||
if self._throttle():
|
||||
return
|
||||
if self._batch_exhausted(result):
|
||||
return
|
||||
if not index_file.is_file():
|
||||
continue
|
||||
try:
|
||||
index_data = json.loads(index_file.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
|
||||
for key_name, entry in list(index_data.items()):
|
||||
if self._throttle():
|
||||
return
|
||||
if self._batch_exhausted(result):
|
||||
return
|
||||
|
||||
rel_dir = index_file.parent.relative_to(meta_root)
|
||||
if rel_dir == Path("."):
|
||||
full_key = key_name
|
||||
else:
|
||||
full_key = rel_dir.as_posix() + "/" + key_name
|
||||
|
||||
object_path = bucket_path / full_key
|
||||
if not object_path.exists():
|
||||
continue
|
||||
|
||||
result.objects_scanned += 1
|
||||
|
||||
meta = entry.get("metadata", {}) if isinstance(entry, dict) else {}
|
||||
stored_etag = meta.get("__etag__")
|
||||
if not stored_etag:
|
||||
continue
|
||||
|
||||
try:
|
||||
actual_etag = _compute_etag(object_path)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
if actual_etag != stored_etag:
|
||||
result.corrupted_objects += 1
|
||||
issue = IntegrityIssue(
|
||||
issue_type="corrupted_object",
|
||||
bucket=bucket_name,
|
||||
key=full_key,
|
||||
detail=f"stored_etag={stored_etag} actual_etag={actual_etag}",
|
||||
)
|
||||
|
||||
if auto_heal and not dry_run:
|
||||
try:
|
||||
stat = object_path.stat()
|
||||
meta["__etag__"] = actual_etag
|
||||
meta["__size__"] = str(stat.st_size)
|
||||
meta["__last_modified__"] = str(stat.st_mtime)
|
||||
index_data[key_name] = {"metadata": meta}
|
||||
self._atomic_write_index(index_file, index_data)
|
||||
issue.healed = True
|
||||
issue.heal_action = "updated etag in index"
|
||||
result.issues_healed += 1
|
||||
except OSError as e:
|
||||
result.errors.append(f"heal corrupted {bucket_name}/{full_key}: {e}")
|
||||
|
||||
self._add_issue(result, issue)
|
||||
except OSError as e:
|
||||
result.errors.append(f"check corrupted {bucket_name}: {e}")
|
||||
|
||||
def _check_orphaned_objects(
|
||||
self, bucket_name: str, result: IntegrityResult, auto_heal: bool, dry_run: bool
|
||||
) -> None:
|
||||
bucket_path = self.storage_root / bucket_name
|
||||
meta_root = self._system_path() / self.SYSTEM_BUCKETS_DIR / bucket_name / self.BUCKET_META_DIR
|
||||
|
||||
try:
|
||||
for entry in bucket_path.rglob("*"):
|
||||
if self._throttle():
|
||||
return
|
||||
if self._batch_exhausted(result):
|
||||
return
|
||||
if not entry.is_file():
|
||||
continue
|
||||
try:
|
||||
rel = entry.relative_to(bucket_path)
|
||||
except ValueError:
|
||||
continue
|
||||
if rel.parts and rel.parts[0] in self.INTERNAL_FOLDERS:
|
||||
continue
|
||||
|
||||
result.objects_scanned += 1
|
||||
full_key = rel.as_posix()
|
||||
key_name = rel.name
|
||||
parent = rel.parent
|
||||
|
||||
if parent == Path("."):
|
||||
index_path = meta_root / "_index.json"
|
||||
else:
|
||||
index_path = meta_root / parent / "_index.json"
|
||||
|
||||
has_entry = False
|
||||
if index_path.exists():
|
||||
try:
|
||||
index_data = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
has_entry = key_name in index_data
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
|
||||
if not has_entry:
|
||||
result.orphaned_objects += 1
|
||||
issue = IntegrityIssue(
|
||||
issue_type="orphaned_object",
|
||||
bucket=bucket_name,
|
||||
key=full_key,
|
||||
detail="file exists without metadata entry",
|
||||
)
|
||||
|
||||
if auto_heal and not dry_run:
|
||||
try:
|
||||
etag = _compute_etag(entry)
|
||||
stat = entry.stat()
|
||||
meta = {
|
||||
"__etag__": etag,
|
||||
"__size__": str(stat.st_size),
|
||||
"__last_modified__": str(stat.st_mtime),
|
||||
}
|
||||
index_data = {}
|
||||
if index_path.exists():
|
||||
try:
|
||||
index_data = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
index_data[key_name] = {"metadata": meta}
|
||||
self._atomic_write_index(index_path, index_data)
|
||||
issue.healed = True
|
||||
issue.heal_action = "created metadata entry"
|
||||
result.issues_healed += 1
|
||||
except OSError as e:
|
||||
result.errors.append(f"heal orphaned {bucket_name}/{full_key}: {e}")
|
||||
|
||||
self._add_issue(result, issue)
|
||||
except OSError as e:
|
||||
result.errors.append(f"check orphaned {bucket_name}: {e}")
|
||||
|
||||
def _check_phantom_metadata(
|
||||
self, bucket_name: str, result: IntegrityResult, auto_heal: bool, dry_run: bool
|
||||
) -> None:
|
||||
bucket_path = self.storage_root / bucket_name
|
||||
meta_root = self._system_path() / self.SYSTEM_BUCKETS_DIR / bucket_name / self.BUCKET_META_DIR
|
||||
|
||||
if not meta_root.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
for index_file in meta_root.rglob("_index.json"):
|
||||
if self._throttle():
|
||||
return
|
||||
if self._batch_exhausted(result):
|
||||
return
|
||||
if not index_file.is_file():
|
||||
continue
|
||||
try:
|
||||
index_data = json.loads(index_file.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
|
||||
keys_to_remove = []
|
||||
for key_name in list(index_data.keys()):
|
||||
if self._batch_exhausted(result):
|
||||
break
|
||||
result.objects_scanned += 1
|
||||
rel_dir = index_file.parent.relative_to(meta_root)
|
||||
if rel_dir == Path("."):
|
||||
full_key = key_name
|
||||
else:
|
||||
full_key = rel_dir.as_posix() + "/" + key_name
|
||||
|
||||
object_path = bucket_path / full_key
|
||||
if not object_path.exists():
|
||||
result.phantom_metadata += 1
|
||||
issue = IntegrityIssue(
|
||||
issue_type="phantom_metadata",
|
||||
bucket=bucket_name,
|
||||
key=full_key,
|
||||
detail="metadata entry without file on disk",
|
||||
)
|
||||
if auto_heal and not dry_run:
|
||||
keys_to_remove.append(key_name)
|
||||
issue.healed = True
|
||||
issue.heal_action = "removed stale index entry"
|
||||
result.issues_healed += 1
|
||||
self._add_issue(result, issue)
|
||||
|
||||
if keys_to_remove and auto_heal and not dry_run:
|
||||
try:
|
||||
for k in keys_to_remove:
|
||||
index_data.pop(k, None)
|
||||
if index_data:
|
||||
self._atomic_write_index(index_file, index_data)
|
||||
else:
|
||||
index_file.unlink(missing_ok=True)
|
||||
except OSError as e:
|
||||
result.errors.append(f"heal phantom {bucket_name}: {e}")
|
||||
except OSError as e:
|
||||
result.errors.append(f"check phantom {bucket_name}: {e}")
|
||||
|
||||
def _check_stale_versions(
|
||||
self, bucket_name: str, result: IntegrityResult, auto_heal: bool, dry_run: bool
|
||||
) -> None:
|
||||
versions_root = self._system_path() / self.SYSTEM_BUCKETS_DIR / bucket_name / self.BUCKET_VERSIONS_DIR
|
||||
|
||||
if not versions_root.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
for key_dir in versions_root.rglob("*"):
|
||||
if self._throttle():
|
||||
return
|
||||
if self._batch_exhausted(result):
|
||||
return
|
||||
if not key_dir.is_dir():
|
||||
continue
|
||||
|
||||
bin_files = {f.stem: f for f in key_dir.glob("*.bin")}
|
||||
json_files = {f.stem: f for f in key_dir.glob("*.json")}
|
||||
|
||||
for stem, bin_file in bin_files.items():
|
||||
if self._batch_exhausted(result):
|
||||
return
|
||||
result.objects_scanned += 1
|
||||
if stem not in json_files:
|
||||
result.stale_versions += 1
|
||||
issue = IntegrityIssue(
|
||||
issue_type="stale_version",
|
||||
bucket=bucket_name,
|
||||
key=f"{key_dir.relative_to(versions_root).as_posix()}/{bin_file.name}",
|
||||
detail="version data without manifest",
|
||||
)
|
||||
if auto_heal and not dry_run:
|
||||
try:
|
||||
bin_file.unlink(missing_ok=True)
|
||||
issue.healed = True
|
||||
issue.heal_action = "removed orphaned version data"
|
||||
result.issues_healed += 1
|
||||
except OSError as e:
|
||||
result.errors.append(f"heal stale version {bin_file}: {e}")
|
||||
self._add_issue(result, issue)
|
||||
|
||||
for stem, json_file in json_files.items():
|
||||
if self._batch_exhausted(result):
|
||||
return
|
||||
result.objects_scanned += 1
|
||||
if stem not in bin_files:
|
||||
result.stale_versions += 1
|
||||
issue = IntegrityIssue(
|
||||
issue_type="stale_version",
|
||||
bucket=bucket_name,
|
||||
key=f"{key_dir.relative_to(versions_root).as_posix()}/{json_file.name}",
|
||||
detail="version manifest without data",
|
||||
)
|
||||
if auto_heal and not dry_run:
|
||||
try:
|
||||
json_file.unlink(missing_ok=True)
|
||||
issue.healed = True
|
||||
issue.heal_action = "removed orphaned version manifest"
|
||||
result.issues_healed += 1
|
||||
except OSError as e:
|
||||
result.errors.append(f"heal stale version {json_file}: {e}")
|
||||
self._add_issue(result, issue)
|
||||
except OSError as e:
|
||||
result.errors.append(f"check stale versions {bucket_name}: {e}")
|
||||
|
||||
def _check_etag_cache(
|
||||
self, bucket_name: str, result: IntegrityResult, auto_heal: bool, dry_run: bool
|
||||
) -> None:
|
||||
etag_index_path = self._system_path() / self.SYSTEM_BUCKETS_DIR / bucket_name / "etag_index.json"
|
||||
|
||||
if not etag_index_path.exists():
|
||||
return
|
||||
|
||||
meta_root = self._system_path() / self.SYSTEM_BUCKETS_DIR / bucket_name / self.BUCKET_META_DIR
|
||||
if not meta_root.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
etag_cache = json.loads(etag_index_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return
|
||||
|
||||
found_mismatch = False
|
||||
|
||||
for full_key, cached_etag in etag_cache.items():
|
||||
if self._batch_exhausted(result):
|
||||
break
|
||||
result.objects_scanned += 1
|
||||
key_path = Path(full_key)
|
||||
key_name = key_path.name
|
||||
parent = key_path.parent
|
||||
|
||||
if parent == Path("."):
|
||||
index_path = meta_root / "_index.json"
|
||||
else:
|
||||
index_path = meta_root / parent / "_index.json"
|
||||
|
||||
if not index_path.exists():
|
||||
continue
|
||||
|
||||
try:
|
||||
index_data = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
|
||||
entry = index_data.get(key_name)
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
meta = entry.get("metadata", {}) if isinstance(entry, dict) else {}
|
||||
stored_etag = meta.get("__etag__")
|
||||
|
||||
if stored_etag and cached_etag != stored_etag:
|
||||
result.etag_cache_inconsistencies += 1
|
||||
found_mismatch = True
|
||||
issue = IntegrityIssue(
|
||||
issue_type="etag_cache_inconsistency",
|
||||
bucket=bucket_name,
|
||||
key=full_key,
|
||||
detail=f"cached_etag={cached_etag} index_etag={stored_etag}",
|
||||
)
|
||||
self._add_issue(result, issue)
|
||||
|
||||
if found_mismatch and auto_heal and not dry_run:
|
||||
try:
|
||||
etag_index_path.unlink(missing_ok=True)
|
||||
for issue in result.issues:
|
||||
if issue.issue_type == "etag_cache_inconsistency" and issue.bucket == bucket_name and not issue.healed:
|
||||
issue.healed = True
|
||||
issue.heal_action = "deleted etag_index.json"
|
||||
result.issues_healed += 1
|
||||
except OSError as e:
|
||||
result.errors.append(f"heal etag cache {bucket_name}: {e}")
|
||||
|
||||
def _check_legacy_metadata(
|
||||
self, bucket_name: str, result: IntegrityResult, auto_heal: bool, dry_run: bool
|
||||
) -> None:
|
||||
legacy_meta_root = self.storage_root / bucket_name / ".meta"
|
||||
if not legacy_meta_root.exists():
|
||||
return
|
||||
|
||||
meta_root = self._system_path() / self.SYSTEM_BUCKETS_DIR / bucket_name / self.BUCKET_META_DIR
|
||||
|
||||
try:
|
||||
for meta_file in legacy_meta_root.rglob("*.meta.json"):
|
||||
if self._throttle():
|
||||
return
|
||||
if self._batch_exhausted(result):
|
||||
return
|
||||
if not meta_file.is_file():
|
||||
continue
|
||||
|
||||
result.objects_scanned += 1
|
||||
try:
|
||||
rel = meta_file.relative_to(legacy_meta_root)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
full_key = rel.as_posix().removesuffix(".meta.json")
|
||||
key_path = Path(full_key)
|
||||
key_name = key_path.name
|
||||
parent = key_path.parent
|
||||
|
||||
if parent == Path("."):
|
||||
index_path = meta_root / "_index.json"
|
||||
else:
|
||||
index_path = meta_root / parent / "_index.json"
|
||||
|
||||
try:
|
||||
legacy_data = json.loads(meta_file.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
|
||||
index_entry = None
|
||||
if index_path.exists():
|
||||
try:
|
||||
index_data = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
index_entry = index_data.get(key_name)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
|
||||
if index_entry is None:
|
||||
result.legacy_metadata_drifts += 1
|
||||
issue = IntegrityIssue(
|
||||
issue_type="legacy_metadata_drift",
|
||||
bucket=bucket_name,
|
||||
key=full_key,
|
||||
detail="unmigrated legacy .meta.json",
|
||||
)
|
||||
|
||||
if auto_heal and not dry_run:
|
||||
try:
|
||||
index_data = {}
|
||||
if index_path.exists():
|
||||
try:
|
||||
index_data = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
index_data[key_name] = {"metadata": legacy_data}
|
||||
self._atomic_write_index(index_path, index_data)
|
||||
meta_file.unlink(missing_ok=True)
|
||||
issue.healed = True
|
||||
issue.heal_action = "migrated to index and deleted legacy file"
|
||||
result.issues_healed += 1
|
||||
except OSError as e:
|
||||
result.errors.append(f"heal legacy {bucket_name}/{full_key}: {e}")
|
||||
|
||||
self._add_issue(result, issue)
|
||||
else:
|
||||
index_meta = index_entry.get("metadata", {}) if isinstance(index_entry, dict) else {}
|
||||
if legacy_data != index_meta:
|
||||
result.legacy_metadata_drifts += 1
|
||||
issue = IntegrityIssue(
|
||||
issue_type="legacy_metadata_drift",
|
||||
bucket=bucket_name,
|
||||
key=full_key,
|
||||
detail="legacy .meta.json differs from index entry",
|
||||
)
|
||||
|
||||
if auto_heal and not dry_run:
|
||||
try:
|
||||
meta_file.unlink(missing_ok=True)
|
||||
issue.healed = True
|
||||
issue.heal_action = "deleted legacy file (index is authoritative)"
|
||||
result.issues_healed += 1
|
||||
except OSError as e:
|
||||
result.errors.append(f"heal legacy drift {bucket_name}/{full_key}: {e}")
|
||||
|
||||
self._add_issue(result, issue)
|
||||
except OSError as e:
|
||||
result.errors.append(f"check legacy meta {bucket_name}: {e}")
|
||||
|
||||
@staticmethod
|
||||
def _atomic_write_index(index_path: Path, data: Dict[str, Any]) -> None:
|
||||
index_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = index_path.with_suffix(".tmp")
|
||||
try:
|
||||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f)
|
||||
os.replace(str(tmp_path), str(index_path))
|
||||
except BaseException:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def get_history(self, limit: int = 50, offset: int = 0) -> List[dict]:
|
||||
records = self.history_store.get_history(limit, offset)
|
||||
return [r.to_dict() for r in records]
|
||||
|
||||
def get_status(self) -> dict:
|
||||
status: Dict[str, Any] = {
|
||||
"enabled": not self._shutdown or self._timer is not None,
|
||||
"running": self._timer is not None and not self._shutdown,
|
||||
"scanning": self._scanning,
|
||||
"interval_hours": self.interval_seconds / 3600.0,
|
||||
"batch_size": self.batch_size,
|
||||
"auto_heal": self.auto_heal,
|
||||
"dry_run": self.dry_run,
|
||||
"io_throttle_ms": round(self._io_throttle * 1000),
|
||||
}
|
||||
if self._scanning and self._scan_start_time is not None:
|
||||
status["scan_elapsed_seconds"] = round(time.time() - self._scan_start_time, 1)
|
||||
status["cursor"] = self.cursor_store.get_info()
|
||||
return status
|
||||
@@ -15,29 +15,23 @@ from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from urllib3.util.connection import create_connection as _urllib3_create_connection
|
||||
|
||||
|
||||
def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
|
||||
"""Check if a URL is safe to make requests to (not internal/private).
|
||||
|
||||
Args:
|
||||
url: The URL to check.
|
||||
allow_internal: If True, allows internal/private IP addresses.
|
||||
Use for self-hosted deployments on internal networks.
|
||||
"""
|
||||
def _resolve_and_check_url(url: str, allow_internal: bool = False) -> Optional[str]:
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname
|
||||
if not hostname:
|
||||
return False
|
||||
return None
|
||||
cloud_metadata_hosts = {
|
||||
"metadata.google.internal",
|
||||
"169.254.169.254",
|
||||
}
|
||||
if hostname.lower() in cloud_metadata_hosts:
|
||||
return False
|
||||
return None
|
||||
if allow_internal:
|
||||
return True
|
||||
return hostname
|
||||
blocked_hosts = {
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
@@ -46,17 +40,46 @@ def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
|
||||
"[::1]",
|
||||
}
|
||||
if hostname.lower() in blocked_hosts:
|
||||
return False
|
||||
return None
|
||||
try:
|
||||
resolved_ip = socket.gethostbyname(hostname)
|
||||
ip = ipaddress.ip_address(resolved_ip)
|
||||
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
|
||||
return False
|
||||
return None
|
||||
return resolved_ip
|
||||
except (socket.gaierror, ValueError):
|
||||
return False
|
||||
return True
|
||||
return None
|
||||
except Exception:
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
|
||||
return _resolve_and_check_url(url, allow_internal) is not None
|
||||
|
||||
|
||||
_dns_pin_lock = threading.Lock()
|
||||
|
||||
|
||||
def _pinned_post(url: str, pinned_ip: str, **kwargs: Any) -> requests.Response:
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname or ""
|
||||
session = requests.Session()
|
||||
original_create = _urllib3_create_connection
|
||||
|
||||
def _create_pinned(address: Any, *args: Any, **kw: Any) -> Any:
|
||||
host, req_port = address
|
||||
if host == hostname:
|
||||
return original_create((pinned_ip, req_port), *args, **kw)
|
||||
return original_create(address, *args, **kw)
|
||||
|
||||
import urllib3.util.connection as _conn_mod
|
||||
with _dns_pin_lock:
|
||||
_conn_mod.create_connection = _create_pinned
|
||||
try:
|
||||
return session.post(url, **kwargs)
|
||||
finally:
|
||||
_conn_mod.create_connection = original_create
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -344,16 +367,18 @@ class NotificationService:
|
||||
self._queue.task_done()
|
||||
|
||||
def _send_notification(self, event: NotificationEvent, destination: WebhookDestination) -> None:
|
||||
if not _is_safe_url(destination.url, allow_internal=self._allow_internal_endpoints):
|
||||
raise RuntimeError(f"Blocked request to cloud metadata service (SSRF protection): {destination.url}")
|
||||
resolved_ip = _resolve_and_check_url(destination.url, allow_internal=self._allow_internal_endpoints)
|
||||
if not resolved_ip:
|
||||
raise RuntimeError(f"Blocked request (SSRF protection): {destination.url}")
|
||||
payload = event.to_s3_event()
|
||||
headers = {"Content-Type": "application/json", **destination.headers}
|
||||
|
||||
last_error = None
|
||||
for attempt in range(destination.retry_count):
|
||||
try:
|
||||
response = requests.post(
|
||||
response = _pinned_post(
|
||||
destination.url,
|
||||
resolved_ip,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=destination.timeout_seconds,
|
||||
|
||||
@@ -2,13 +2,17 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
MAX_LATENCY_SAMPLES = 5000
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -22,6 +26,17 @@ class OperationStats:
|
||||
latency_max_ms: float = 0.0
|
||||
bytes_in: int = 0
|
||||
bytes_out: int = 0
|
||||
latency_samples: List[float] = field(default_factory=list)
|
||||
|
||||
@staticmethod
|
||||
def _compute_percentile(sorted_data: List[float], p: float) -> float:
|
||||
if not sorted_data:
|
||||
return 0.0
|
||||
k = (len(sorted_data) - 1) * (p / 100.0)
|
||||
f = int(k)
|
||||
c = min(f + 1, len(sorted_data) - 1)
|
||||
d = k - f
|
||||
return sorted_data[f] + d * (sorted_data[c] - sorted_data[f])
|
||||
|
||||
def record(self, latency_ms: float, success: bool, bytes_in: int = 0, bytes_out: int = 0) -> None:
|
||||
self.count += 1
|
||||
@@ -36,10 +51,17 @@ class OperationStats:
|
||||
self.latency_max_ms = latency_ms
|
||||
self.bytes_in += bytes_in
|
||||
self.bytes_out += bytes_out
|
||||
if len(self.latency_samples) < MAX_LATENCY_SAMPLES:
|
||||
self.latency_samples.append(latency_ms)
|
||||
else:
|
||||
j = random.randint(0, self.count - 1)
|
||||
if j < MAX_LATENCY_SAMPLES:
|
||||
self.latency_samples[j] = latency_ms
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
avg_latency = self.latency_sum_ms / self.count if self.count > 0 else 0.0
|
||||
min_latency = self.latency_min_ms if self.latency_min_ms != float("inf") else 0.0
|
||||
sorted_latencies = sorted(self.latency_samples)
|
||||
return {
|
||||
"count": self.count,
|
||||
"success_count": self.success_count,
|
||||
@@ -47,6 +69,9 @@ class OperationStats:
|
||||
"latency_avg_ms": round(avg_latency, 2),
|
||||
"latency_min_ms": round(min_latency, 2),
|
||||
"latency_max_ms": round(self.latency_max_ms, 2),
|
||||
"latency_p50_ms": round(self._compute_percentile(sorted_latencies, 50), 2),
|
||||
"latency_p95_ms": round(self._compute_percentile(sorted_latencies, 95), 2),
|
||||
"latency_p99_ms": round(self._compute_percentile(sorted_latencies, 99), 2),
|
||||
"bytes_in": self.bytes_in,
|
||||
"bytes_out": self.bytes_out,
|
||||
}
|
||||
@@ -62,6 +87,11 @@ class OperationStats:
|
||||
self.latency_max_ms = other.latency_max_ms
|
||||
self.bytes_in += other.bytes_in
|
||||
self.bytes_out += other.bytes_out
|
||||
combined = self.latency_samples + other.latency_samples
|
||||
if len(combined) > MAX_LATENCY_SAMPLES:
|
||||
random.shuffle(combined)
|
||||
combined = combined[:MAX_LATENCY_SAMPLES]
|
||||
self.latency_samples = combined
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -109,8 +139,8 @@ class OperationMetricsCollector:
|
||||
self.interval_seconds = interval_minutes * 60
|
||||
self.retention_hours = retention_hours
|
||||
self._lock = threading.Lock()
|
||||
self._by_method: Dict[str, OperationStats] = {}
|
||||
self._by_endpoint: Dict[str, OperationStats] = {}
|
||||
self._by_method: Dict[str, OperationStats] = defaultdict(OperationStats)
|
||||
self._by_endpoint: Dict[str, OperationStats] = defaultdict(OperationStats)
|
||||
self._by_status_class: Dict[str, int] = {}
|
||||
self._error_codes: Dict[str, int] = {}
|
||||
self._totals = OperationStats()
|
||||
@@ -182,8 +212,8 @@ class OperationMetricsCollector:
|
||||
self._prune_old_snapshots()
|
||||
self._save_history()
|
||||
|
||||
self._by_method.clear()
|
||||
self._by_endpoint.clear()
|
||||
self._by_method = defaultdict(OperationStats)
|
||||
self._by_endpoint = defaultdict(OperationStats)
|
||||
self._by_status_class.clear()
|
||||
self._error_codes.clear()
|
||||
self._totals = OperationStats()
|
||||
@@ -203,12 +233,7 @@ class OperationMetricsCollector:
|
||||
status_class = f"{status_code // 100}xx"
|
||||
|
||||
with self._lock:
|
||||
if method not in self._by_method:
|
||||
self._by_method[method] = OperationStats()
|
||||
self._by_method[method].record(latency_ms, success, bytes_in, bytes_out)
|
||||
|
||||
if endpoint_type not in self._by_endpoint:
|
||||
self._by_endpoint[endpoint_type] = OperationStats()
|
||||
self._by_endpoint[endpoint_type].record(latency_ms, success, bytes_in, bytes_out)
|
||||
|
||||
self._by_status_class[status_class] = self._by_status_class.get(status_class, 0) + 1
|
||||
|
||||
602
app/s3_api.py
602
app/s3_api.py
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import re
|
||||
@@ -16,6 +17,13 @@ from urllib.parse import quote, urlencode, urlparse, unquote
|
||||
from xml.etree.ElementTree import Element, SubElement, tostring, ParseError
|
||||
from defusedxml.ElementTree import fromstring
|
||||
|
||||
try:
|
||||
import myfsio_core as _rc
|
||||
_HAS_RUST = True
|
||||
except ImportError:
|
||||
_rc = None
|
||||
_HAS_RUST = False
|
||||
|
||||
from flask import Blueprint, Response, current_app, jsonify, request, g
|
||||
from werkzeug.http import http_date
|
||||
|
||||
@@ -77,6 +85,9 @@ def _bucket_policies() -> BucketPolicyStore:
|
||||
|
||||
|
||||
def _build_policy_context() -> Dict[str, Any]:
|
||||
cached = getattr(g, "_policy_context", None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
ctx: Dict[str, Any] = {}
|
||||
if request.headers.get("Referer"):
|
||||
ctx["aws:Referer"] = request.headers.get("Referer")
|
||||
@@ -90,6 +101,7 @@ def _build_policy_context() -> Dict[str, Any]:
|
||||
ctx["aws:SecureTransport"] = str(request.is_secure).lower()
|
||||
if request.headers.get("User-Agent"):
|
||||
ctx["aws:UserAgent"] = request.headers.get("User-Agent")
|
||||
g._policy_context = ctx
|
||||
return ctx
|
||||
|
||||
|
||||
@@ -191,11 +203,16 @@ _SIGNING_KEY_CACHE_MAX_SIZE = 256
|
||||
|
||||
|
||||
def clear_signing_key_cache() -> None:
|
||||
if _HAS_RUST:
|
||||
_rc.clear_signing_key_cache()
|
||||
with _SIGNING_KEY_CACHE_LOCK:
|
||||
_SIGNING_KEY_CACHE.clear()
|
||||
|
||||
|
||||
def _get_signature_key(key: str, date_stamp: str, region_name: str, service_name: str) -> bytes:
|
||||
if _HAS_RUST:
|
||||
return bytes(_rc.derive_signing_key(key, date_stamp, region_name, service_name))
|
||||
|
||||
cache_key = (key, date_stamp, region_name, service_name)
|
||||
now = time.time()
|
||||
|
||||
@@ -254,39 +271,6 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
|
||||
if not secret_key:
|
||||
raise IamError("SignatureDoesNotMatch")
|
||||
|
||||
method = req.method
|
||||
canonical_uri = _get_canonical_uri(req)
|
||||
|
||||
query_args = []
|
||||
for key, value in req.args.items(multi=True):
|
||||
query_args.append((key, value))
|
||||
query_args.sort(key=lambda x: (x[0], x[1]))
|
||||
|
||||
canonical_query_parts = []
|
||||
for k, v in query_args:
|
||||
canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
|
||||
canonical_query_string = "&".join(canonical_query_parts)
|
||||
|
||||
signed_headers_list = signed_headers_str.split(";")
|
||||
canonical_headers_parts = []
|
||||
for header in signed_headers_list:
|
||||
header_val = req.headers.get(header)
|
||||
if header_val is None:
|
||||
header_val = ""
|
||||
|
||||
if header.lower() == 'expect' and header_val == "":
|
||||
header_val = "100-continue"
|
||||
|
||||
header_val = " ".join(header_val.split())
|
||||
canonical_headers_parts.append(f"{header.lower()}:{header_val}\n")
|
||||
canonical_headers = "".join(canonical_headers_parts)
|
||||
|
||||
payload_hash = req.headers.get("X-Amz-Content-Sha256")
|
||||
if not payload_hash:
|
||||
payload_hash = hashlib.sha256(req.get_data()).hexdigest()
|
||||
|
||||
canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}"
|
||||
|
||||
amz_date = req.headers.get("X-Amz-Date") or req.headers.get("Date")
|
||||
if not amz_date:
|
||||
raise IamError("Missing Date header")
|
||||
@@ -312,14 +296,50 @@ def _verify_sigv4_header(req: Any, auth_header: str) -> Principal | None:
|
||||
if not required_headers.issubset(signed_headers_set):
|
||||
raise IamError("Required headers not signed")
|
||||
|
||||
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
|
||||
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
|
||||
signing_key = _get_signature_key(secret_key, date_stamp, region, service)
|
||||
calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
canonical_uri = _get_canonical_uri(req)
|
||||
payload_hash = req.headers.get("X-Amz-Content-Sha256") or "UNSIGNED-PAYLOAD"
|
||||
|
||||
if _HAS_RUST:
|
||||
query_params = list(req.args.items(multi=True))
|
||||
header_values = []
|
||||
for h in signed_headers_str.split(";"):
|
||||
val = req.headers.get(h) or ""
|
||||
if h.lower() == "expect" and val == "":
|
||||
val = "100-continue"
|
||||
header_values.append((h, val))
|
||||
if not _rc.verify_sigv4_signature(
|
||||
req.method, canonical_uri, query_params, signed_headers_str,
|
||||
header_values, payload_hash, amz_date, date_stamp, region,
|
||||
service, secret_key, signature,
|
||||
):
|
||||
raise IamError("SignatureDoesNotMatch")
|
||||
else:
|
||||
method = req.method
|
||||
query_args = sorted(req.args.items(multi=True), key=lambda x: (x[0], x[1]))
|
||||
canonical_query_parts = []
|
||||
for k, v in query_args:
|
||||
canonical_query_parts.append(f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}")
|
||||
canonical_query_string = "&".join(canonical_query_parts)
|
||||
|
||||
signed_headers_list = signed_headers_str.split(";")
|
||||
canonical_headers_parts = []
|
||||
for header in signed_headers_list:
|
||||
header_val = req.headers.get(header)
|
||||
if header_val is None:
|
||||
header_val = ""
|
||||
if header.lower() == 'expect' and header_val == "":
|
||||
header_val = "100-continue"
|
||||
header_val = " ".join(header_val.split())
|
||||
canonical_headers_parts.append(f"{header.lower()}:{header_val}\n")
|
||||
canonical_headers = "".join(canonical_headers_parts)
|
||||
|
||||
canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers_str}\n{payload_hash}"
|
||||
|
||||
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
|
||||
signing_key = _get_signature_key(secret_key, date_stamp, region, service)
|
||||
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
|
||||
calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
if not hmac.compare_digest(calculated_signature, signature):
|
||||
if current_app.config.get("DEBUG_SIGV4"):
|
||||
logger.warning("SigV4 signature mismatch for %s %s", method, req.path)
|
||||
raise IamError("SignatureDoesNotMatch")
|
||||
|
||||
session_token = req.headers.get("X-Amz-Security-Token")
|
||||
@@ -351,12 +371,19 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
|
||||
raise IamError("Invalid Date format")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900))
|
||||
if req_time > now + tolerance:
|
||||
raise IamError("Request date is too far in the future")
|
||||
try:
|
||||
expires_seconds = int(expires)
|
||||
if expires_seconds <= 0:
|
||||
raise IamError("Invalid Expires value: must be positive")
|
||||
except ValueError:
|
||||
raise IamError("Invalid Expires value: must be an integer")
|
||||
min_expiry = current_app.config.get("PRESIGNED_URL_MIN_EXPIRY_SECONDS", 1)
|
||||
max_expiry = current_app.config.get("PRESIGNED_URL_MAX_EXPIRY_SECONDS", 604800)
|
||||
if expires_seconds < min_expiry or expires_seconds > max_expiry:
|
||||
raise IamError(f"Expiration must be between {min_expiry} second(s) and {max_expiry} seconds")
|
||||
if now > req_time + timedelta(seconds=expires_seconds):
|
||||
raise IamError("Request expired")
|
||||
|
||||
@@ -364,9 +391,24 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
|
||||
if not secret_key:
|
||||
raise IamError("Invalid access key")
|
||||
|
||||
method = req.method
|
||||
canonical_uri = _get_canonical_uri(req)
|
||||
|
||||
if _HAS_RUST:
|
||||
query_params = [(k, v) for k, v in req.args.items(multi=True) if k != "X-Amz-Signature"]
|
||||
header_values = []
|
||||
for h in signed_headers_str.split(";"):
|
||||
val = req.headers.get(h) or ""
|
||||
if h.lower() == "expect" and val == "":
|
||||
val = "100-continue"
|
||||
header_values.append((h, val))
|
||||
if not _rc.verify_sigv4_signature(
|
||||
req.method, canonical_uri, query_params, signed_headers_str,
|
||||
header_values, "UNSIGNED-PAYLOAD", amz_date, date_stamp, region,
|
||||
service, secret_key, signature,
|
||||
):
|
||||
raise IamError("SignatureDoesNotMatch")
|
||||
else:
|
||||
method = req.method
|
||||
query_args = []
|
||||
for key, value in req.args.items(multi=True):
|
||||
if key != "X-Amz-Signature":
|
||||
@@ -399,19 +441,11 @@ def _verify_sigv4_query(req: Any) -> Principal | None:
|
||||
payload_hash
|
||||
])
|
||||
|
||||
algorithm = "AWS4-HMAC-SHA256"
|
||||
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
|
||||
hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
|
||||
string_to_sign = "\n".join([
|
||||
algorithm,
|
||||
amz_date,
|
||||
credential_scope,
|
||||
hashed_request
|
||||
])
|
||||
|
||||
signing_key = _get_signature_key(secret_key, date_stamp, region, service)
|
||||
hashed_request = hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
|
||||
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashed_request}"
|
||||
calculated_signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
|
||||
if not hmac.compare_digest(calculated_signature, signature):
|
||||
raise IamError("SignatureDoesNotMatch")
|
||||
|
||||
@@ -464,7 +498,7 @@ def _authorize_action(principal: Principal | None, bucket_name: str | None, acti
|
||||
iam_error: IamError | None = None
|
||||
if principal is not None:
|
||||
try:
|
||||
_iam().authorize(principal, bucket_name, action)
|
||||
_iam().authorize(principal, bucket_name, action, object_key=object_key)
|
||||
iam_allowed = True
|
||||
except IamError as exc:
|
||||
iam_error = exc
|
||||
@@ -572,7 +606,11 @@ def _validate_presigned_request(action: str, bucket_name: str, object_key: str)
|
||||
request_time = datetime.strptime(amz_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)
|
||||
except ValueError as exc:
|
||||
raise IamError("Invalid X-Amz-Date") from exc
|
||||
if datetime.now(timezone.utc) > request_time + timedelta(seconds=expiry):
|
||||
now = datetime.now(timezone.utc)
|
||||
tolerance = timedelta(seconds=current_app.config.get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900))
|
||||
if request_time > now + tolerance:
|
||||
raise IamError("Request date is too far in the future")
|
||||
if now > request_time + timedelta(seconds=expiry):
|
||||
raise IamError("Presigned URL expired")
|
||||
|
||||
signed_headers_list = [header.strip().lower() for header in signed_headers.split(";") if header]
|
||||
@@ -648,7 +686,7 @@ def _extract_request_metadata() -> Dict[str, str]:
|
||||
for header, value in request.headers.items():
|
||||
if header.lower().startswith("x-amz-meta-"):
|
||||
key = header[11:]
|
||||
if key:
|
||||
if key and not (key.startswith("__") and key.endswith("__")):
|
||||
metadata[key] = value
|
||||
return metadata
|
||||
|
||||
@@ -972,7 +1010,7 @@ def _render_encryption_document(config: dict[str, Any]) -> Element:
|
||||
return root
|
||||
|
||||
|
||||
def _stream_file(path, chunk_size: int = 64 * 1024):
|
||||
def _stream_file(path, chunk_size: int = 256 * 1024):
|
||||
with path.open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(chunk_size)
|
||||
@@ -991,20 +1029,78 @@ def _method_not_allowed(allowed: list[str]) -> Response:
|
||||
return response
|
||||
|
||||
|
||||
def _check_conditional_headers(etag: str, last_modified: float | None) -> Response | None:
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
if_match = request.headers.get("If-Match")
|
||||
if if_match:
|
||||
if if_match.strip() != "*":
|
||||
match_etags = [e.strip().strip('"') for e in if_match.split(",")]
|
||||
if etag not in match_etags:
|
||||
return Response(status=412)
|
||||
|
||||
if_unmodified = request.headers.get("If-Unmodified-Since")
|
||||
if not if_match and if_unmodified and last_modified is not None:
|
||||
try:
|
||||
dt = parsedate_to_datetime(if_unmodified)
|
||||
obj_dt = datetime.fromtimestamp(last_modified, timezone.utc)
|
||||
if obj_dt > dt:
|
||||
return Response(status=412)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if_none_match = request.headers.get("If-None-Match")
|
||||
if if_none_match:
|
||||
if if_none_match.strip() == "*":
|
||||
resp = Response(status=304)
|
||||
resp.headers["ETag"] = f'"{etag}"'
|
||||
if last_modified is not None:
|
||||
resp.headers["Last-Modified"] = http_date(last_modified)
|
||||
return resp
|
||||
none_match_etags = [e.strip().strip('"') for e in if_none_match.split(",")]
|
||||
if etag in none_match_etags:
|
||||
resp = Response(status=304)
|
||||
resp.headers["ETag"] = f'"{etag}"'
|
||||
if last_modified is not None:
|
||||
resp.headers["Last-Modified"] = http_date(last_modified)
|
||||
return resp
|
||||
|
||||
if_modified = request.headers.get("If-Modified-Since")
|
||||
if not if_none_match and if_modified and last_modified is not None:
|
||||
try:
|
||||
dt = parsedate_to_datetime(if_modified)
|
||||
obj_dt = datetime.fromtimestamp(last_modified, timezone.utc)
|
||||
if obj_dt <= dt:
|
||||
resp = Response(status=304)
|
||||
resp.headers["ETag"] = f'"{etag}"'
|
||||
resp.headers["Last-Modified"] = http_date(last_modified)
|
||||
return resp
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _apply_object_headers(
|
||||
response: Response,
|
||||
*,
|
||||
file_stat,
|
||||
metadata: Dict[str, str] | None,
|
||||
etag: str,
|
||||
size_override: int | None = None,
|
||||
mtime_override: float | None = None,
|
||||
) -> None:
|
||||
if file_stat is not None:
|
||||
if response.status_code != 206:
|
||||
response.headers["Content-Length"] = str(file_stat.st_size)
|
||||
response.headers["Last-Modified"] = http_date(file_stat.st_mtime)
|
||||
effective_size = size_override if size_override is not None else (file_stat.st_size if file_stat is not None else None)
|
||||
effective_mtime = mtime_override if mtime_override is not None else (file_stat.st_mtime if file_stat is not None else None)
|
||||
if effective_size is not None and response.status_code != 206:
|
||||
response.headers["Content-Length"] = str(effective_size)
|
||||
if effective_mtime is not None:
|
||||
response.headers["Last-Modified"] = http_date(effective_mtime)
|
||||
response.headers["ETag"] = f'"{etag}"'
|
||||
response.headers["Accept-Ranges"] = "bytes"
|
||||
for key, value in (metadata or {}).items():
|
||||
if key.startswith("__") and key.endswith("__"):
|
||||
continue
|
||||
safe_value = _sanitize_header_value(str(value))
|
||||
response.headers[f"X-Amz-Meta-{key}"] = safe_value
|
||||
|
||||
@@ -1025,7 +1121,9 @@ def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None:
|
||||
"logging": _bucket_logging_handler,
|
||||
"uploads": _bucket_uploads_handler,
|
||||
"policy": _bucket_policy_handler,
|
||||
"policyStatus": _bucket_policy_status_handler,
|
||||
"replication": _bucket_replication_handler,
|
||||
"website": _bucket_website_handler,
|
||||
}
|
||||
requested = [key for key in handlers if key in request.args]
|
||||
if not requested:
|
||||
@@ -1047,7 +1145,7 @@ def _bucket_versioning_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "versioning")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
storage = _storage()
|
||||
@@ -1094,7 +1192,7 @@ def _bucket_tagging_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "tagging")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
storage = _storage()
|
||||
@@ -1259,7 +1357,7 @@ def _bucket_cors_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "cors")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
storage = _storage()
|
||||
@@ -1306,13 +1404,13 @@ def _bucket_cors_handler(bucket_name: str) -> Response:
|
||||
|
||||
|
||||
def _bucket_encryption_handler(bucket_name: str) -> Response:
|
||||
if request.method not in {"GET", "PUT"}:
|
||||
return _method_not_allowed(["GET", "PUT"])
|
||||
if request.method not in {"GET", "PUT", "DELETE"}:
|
||||
return _method_not_allowed(["GET", "PUT", "DELETE"])
|
||||
principal, error = _require_principal()
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "encryption")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
storage = _storage()
|
||||
@@ -1328,6 +1426,13 @@ def _bucket_encryption_handler(bucket_name: str) -> Response:
|
||||
404,
|
||||
)
|
||||
return _xml_response(_render_encryption_document(config))
|
||||
if request.method == "DELETE":
|
||||
try:
|
||||
storage.set_bucket_encryption(bucket_name, None)
|
||||
except StorageError as exc:
|
||||
return _error_response("NoSuchBucket", str(exc), 404)
|
||||
current_app.logger.info("Bucket encryption deleted", extra={"bucket": bucket_name})
|
||||
return Response(status=204)
|
||||
ct_error = _require_xml_content_type()
|
||||
if ct_error:
|
||||
return ct_error
|
||||
@@ -1380,7 +1485,7 @@ def _bucket_acl_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "share")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
storage = _storage()
|
||||
@@ -1424,6 +1529,99 @@ def _bucket_acl_handler(bucket_name: str) -> Response:
|
||||
return _xml_response(root)
|
||||
|
||||
|
||||
def _object_acl_handler(bucket_name: str, object_key: str) -> Response:
|
||||
from .acl import create_canned_acl, GRANTEE_ALL_USERS, GRANTEE_AUTHENTICATED_USERS
|
||||
|
||||
if request.method not in {"GET", "PUT"}:
|
||||
return _method_not_allowed(["GET", "PUT"])
|
||||
storage = _storage()
|
||||
try:
|
||||
path = storage.get_object_path(bucket_name, object_key)
|
||||
except (StorageError, FileNotFoundError):
|
||||
return _error_response("NoSuchKey", "Object not found", 404)
|
||||
|
||||
if request.method == "PUT":
|
||||
principal, error = _object_principal("write", bucket_name, object_key)
|
||||
if error:
|
||||
return error
|
||||
owner_id = principal.access_key if principal else "anonymous"
|
||||
canned_acl = request.headers.get("x-amz-acl", "private")
|
||||
acl = create_canned_acl(canned_acl, owner_id)
|
||||
acl_service = _acl()
|
||||
metadata = storage.get_object_metadata(bucket_name, object_key)
|
||||
metadata.update(acl_service.create_object_acl_metadata(acl))
|
||||
safe_key = storage._sanitize_object_key(object_key, storage._object_key_max_length_bytes)
|
||||
storage._write_metadata(bucket_name, safe_key, metadata)
|
||||
current_app.logger.info("Object ACL set", extra={"bucket": bucket_name, "key": object_key, "acl": canned_acl})
|
||||
return Response(status=200)
|
||||
|
||||
principal, error = _object_principal("read", bucket_name, object_key)
|
||||
if error:
|
||||
return error
|
||||
owner_id = principal.access_key if principal else "anonymous"
|
||||
acl_service = _acl()
|
||||
metadata = storage.get_object_metadata(bucket_name, object_key)
|
||||
acl = acl_service.get_object_acl(bucket_name, object_key, metadata)
|
||||
if not acl:
|
||||
acl = create_canned_acl("private", owner_id)
|
||||
|
||||
root = Element("AccessControlPolicy")
|
||||
owner_el = SubElement(root, "Owner")
|
||||
SubElement(owner_el, "ID").text = acl.owner
|
||||
SubElement(owner_el, "DisplayName").text = acl.owner
|
||||
acl_el = SubElement(root, "AccessControlList")
|
||||
for grant in acl.grants:
|
||||
grant_el = SubElement(acl_el, "Grant")
|
||||
grantee = SubElement(grant_el, "Grantee")
|
||||
if grant.grantee == GRANTEE_ALL_USERS:
|
||||
grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group")
|
||||
SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
elif grant.grantee == GRANTEE_AUTHENTICATED_USERS:
|
||||
grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "Group")
|
||||
SubElement(grantee, "URI").text = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
|
||||
else:
|
||||
grantee.set("{http://www.w3.org/2001/XMLSchema-instance}type", "CanonicalUser")
|
||||
SubElement(grantee, "ID").text = grant.grantee
|
||||
SubElement(grantee, "DisplayName").text = grant.grantee
|
||||
SubElement(grant_el, "Permission").text = grant.permission
|
||||
return _xml_response(root)
|
||||
|
||||
|
||||
def _object_attributes_handler(bucket_name: str, object_key: str) -> Response:
|
||||
if request.method != "GET":
|
||||
return _method_not_allowed(["GET"])
|
||||
principal, error = _object_principal("read", bucket_name, object_key)
|
||||
if error:
|
||||
return error
|
||||
storage = _storage()
|
||||
try:
|
||||
path = storage.get_object_path(bucket_name, object_key)
|
||||
file_stat = path.stat()
|
||||
metadata = storage.get_object_metadata(bucket_name, object_key)
|
||||
except (StorageError, FileNotFoundError):
|
||||
return _error_response("NoSuchKey", "Object not found", 404)
|
||||
|
||||
requested = request.headers.get("x-amz-object-attributes", "")
|
||||
attrs = {a.strip() for a in requested.split(",") if a.strip()}
|
||||
|
||||
root = Element("GetObjectAttributesResponse")
|
||||
if "ETag" in attrs:
|
||||
etag = metadata.get("__etag__") or storage._compute_etag(path)
|
||||
SubElement(root, "ETag").text = etag
|
||||
if "StorageClass" in attrs:
|
||||
SubElement(root, "StorageClass").text = "STANDARD"
|
||||
if "ObjectSize" in attrs:
|
||||
SubElement(root, "ObjectSize").text = str(file_stat.st_size)
|
||||
if "Checksum" in attrs:
|
||||
SubElement(root, "Checksum")
|
||||
if "ObjectParts" in attrs:
|
||||
SubElement(root, "ObjectParts")
|
||||
|
||||
response = _xml_response(root)
|
||||
response.headers["Last-Modified"] = http_date(file_stat.st_mtime)
|
||||
return response
|
||||
|
||||
|
||||
def _bucket_list_versions_handler(bucket_name: str) -> Response:
|
||||
"""Handle ListObjectVersions (GET /<bucket>?versions)."""
|
||||
if request.method != "GET":
|
||||
@@ -1535,7 +1733,7 @@ def _bucket_lifecycle_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "lifecycle")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
|
||||
@@ -1699,7 +1897,7 @@ def _bucket_quota_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "quota")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
|
||||
@@ -1776,7 +1974,7 @@ def _bucket_object_lock_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "object_lock")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
|
||||
@@ -1822,7 +2020,7 @@ def _bucket_notification_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "notification")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
|
||||
@@ -1918,7 +2116,7 @@ def _bucket_logging_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "logging")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
|
||||
@@ -2060,7 +2258,7 @@ def _object_retention_handler(bucket_name: str, object_key: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "write" if request.method == "PUT" else "read", object_key=object_key)
|
||||
_authorize_action(principal, bucket_name, "object_lock", object_key=object_key)
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
|
||||
@@ -2136,7 +2334,7 @@ def _object_legal_hold_handler(bucket_name: str, object_key: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "write" if request.method == "PUT" else "read", object_key=object_key)
|
||||
_authorize_action(principal, bucket_name, "object_lock", object_key=object_key)
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
|
||||
@@ -2331,7 +2529,7 @@ def _post_object(bucket_name: str) -> Response:
|
||||
for field_name, value in request.form.items():
|
||||
if field_name.lower().startswith("x-amz-meta-"):
|
||||
key = field_name[11:]
|
||||
if key:
|
||||
if key and not (key.startswith("__") and key.endswith("__")):
|
||||
metadata[key] = value
|
||||
try:
|
||||
meta = storage.put_object(bucket_name, object_key, file.stream, metadata=metadata or None)
|
||||
@@ -2345,6 +2543,10 @@ def _post_object(bucket_name: str) -> Response:
|
||||
if success_action_redirect:
|
||||
allowed_hosts = current_app.config.get("ALLOWED_REDIRECT_HOSTS", [])
|
||||
if not allowed_hosts:
|
||||
current_app.logger.warning(
|
||||
"ALLOWED_REDIRECT_HOSTS not configured, falling back to request Host header. "
|
||||
"Set ALLOWED_REDIRECT_HOSTS for production deployments."
|
||||
)
|
||||
allowed_hosts = [request.host]
|
||||
parsed = urlparse(success_action_redirect)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
@@ -2465,7 +2667,7 @@ def bucket_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "write")
|
||||
_authorize_action(principal, bucket_name, "create_bucket")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
try:
|
||||
@@ -2482,7 +2684,7 @@ def bucket_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "delete")
|
||||
_authorize_action(principal, bucket_name, "delete_bucket")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
try:
|
||||
@@ -2531,54 +2733,43 @@ def bucket_handler(bucket_name: str) -> Response:
|
||||
else:
|
||||
effective_start = marker
|
||||
|
||||
fetch_keys = max_keys * 10 if delimiter else max_keys
|
||||
try:
|
||||
if delimiter:
|
||||
shallow_result = storage.list_objects_shallow(
|
||||
bucket_name,
|
||||
prefix=prefix,
|
||||
delimiter=delimiter,
|
||||
max_keys=max_keys,
|
||||
continuation_token=effective_start or None,
|
||||
)
|
||||
objects = shallow_result.objects
|
||||
common_prefixes = shallow_result.common_prefixes
|
||||
is_truncated = shallow_result.is_truncated
|
||||
|
||||
next_marker = shallow_result.next_continuation_token or ""
|
||||
next_continuation_token = ""
|
||||
if is_truncated and next_marker and list_type == "2":
|
||||
next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
|
||||
else:
|
||||
list_result = storage.list_objects(
|
||||
bucket_name,
|
||||
max_keys=fetch_keys,
|
||||
max_keys=max_keys,
|
||||
continuation_token=effective_start or None,
|
||||
prefix=prefix or None,
|
||||
)
|
||||
objects = list_result.objects
|
||||
except StorageError as exc:
|
||||
return _error_response("NoSuchBucket", str(exc), 404)
|
||||
|
||||
common_prefixes: list[str] = []
|
||||
filtered_objects: list = []
|
||||
if delimiter:
|
||||
seen_prefixes: set[str] = set()
|
||||
for obj in objects:
|
||||
key_after_prefix = obj.key[len(prefix):] if prefix else obj.key
|
||||
if delimiter in key_after_prefix:
|
||||
common_prefix = prefix + key_after_prefix.split(delimiter)[0] + delimiter
|
||||
if common_prefix not in seen_prefixes:
|
||||
seen_prefixes.add(common_prefix)
|
||||
common_prefixes.append(common_prefix)
|
||||
else:
|
||||
filtered_objects.append(obj)
|
||||
objects = filtered_objects
|
||||
common_prefixes = sorted(common_prefixes)
|
||||
|
||||
total_items = len(objects) + len(common_prefixes)
|
||||
is_truncated = total_items > max_keys or list_result.is_truncated
|
||||
|
||||
if len(objects) >= max_keys:
|
||||
objects = objects[:max_keys]
|
||||
common_prefixes = []
|
||||
else:
|
||||
remaining = max_keys - len(objects)
|
||||
common_prefixes = common_prefixes[:remaining]
|
||||
is_truncated = list_result.is_truncated
|
||||
|
||||
next_marker = ""
|
||||
next_continuation_token = ""
|
||||
if is_truncated:
|
||||
if objects:
|
||||
next_marker = objects[-1].key
|
||||
elif common_prefixes:
|
||||
next_marker = common_prefixes[-1].rstrip(delimiter) if delimiter else common_prefixes[-1]
|
||||
|
||||
if list_type == "2" and next_marker:
|
||||
next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
|
||||
except StorageError as exc:
|
||||
return _error_response("NoSuchBucket", str(exc), 404)
|
||||
|
||||
if list_type == "2":
|
||||
root = Element("ListBucketResult")
|
||||
@@ -2654,6 +2845,12 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
if "legal-hold" in request.args:
|
||||
return _object_legal_hold_handler(bucket_name, object_key)
|
||||
|
||||
if "acl" in request.args:
|
||||
return _object_acl_handler(bucket_name, object_key)
|
||||
|
||||
if "attributes" in request.args:
|
||||
return _object_attributes_handler(bucket_name, object_key)
|
||||
|
||||
if request.method == "POST":
|
||||
if "uploads" in request.args:
|
||||
return _initiate_multipart_upload(bucket_name, object_key)
|
||||
@@ -2693,6 +2890,8 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
if validation_error:
|
||||
return _error_response("InvalidArgument", validation_error, 400)
|
||||
|
||||
metadata["__content_type__"] = content_type or mimetypes.guess_type(object_key)[0] or "application/octet-stream"
|
||||
|
||||
try:
|
||||
meta = storage.put_object(
|
||||
bucket_name,
|
||||
@@ -2707,6 +2906,19 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
if "Bucket" in message:
|
||||
return _error_response("NoSuchBucket", message, 404)
|
||||
return _error_response("InvalidArgument", message, 400)
|
||||
|
||||
content_md5 = request.headers.get("Content-MD5")
|
||||
if content_md5 and meta.etag:
|
||||
try:
|
||||
expected_md5 = base64.b64decode(content_md5).hex()
|
||||
except Exception:
|
||||
storage.delete_object(bucket_name, object_key)
|
||||
return _error_response("InvalidDigest", "Content-MD5 header is not valid base64", 400)
|
||||
if expected_md5 != meta.etag:
|
||||
storage.delete_object(bucket_name, object_key)
|
||||
return _error_response("BadDigest", "The Content-MD5 you specified did not match what we received", 400)
|
||||
|
||||
if current_app.logger.isEnabledFor(logging.INFO):
|
||||
current_app.logger.info(
|
||||
"Object uploaded",
|
||||
extra={"bucket": bucket_name, "key": object_key, "size": meta.size},
|
||||
@@ -2744,10 +2956,27 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
except StorageError as exc:
|
||||
return _error_response("NoSuchKey", str(exc), 404)
|
||||
metadata = storage.get_object_metadata(bucket_name, object_key)
|
||||
mimetype = mimetypes.guess_type(object_key)[0] or "application/octet-stream"
|
||||
mimetype = metadata.get("__content_type__") or mimetypes.guess_type(object_key)[0] or "application/octet-stream"
|
||||
|
||||
is_encrypted = "x-amz-server-side-encryption" in metadata
|
||||
|
||||
cond_etag = metadata.get("__etag__")
|
||||
if not cond_etag and not is_encrypted:
|
||||
try:
|
||||
cond_etag = storage._compute_etag(path)
|
||||
except OSError:
|
||||
cond_etag = None
|
||||
if cond_etag:
|
||||
cond_mtime = float(metadata["__last_modified__"]) if "__last_modified__" in metadata else None
|
||||
if cond_mtime is None:
|
||||
try:
|
||||
cond_mtime = path.stat().st_mtime
|
||||
except OSError:
|
||||
pass
|
||||
cond_resp = _check_conditional_headers(cond_etag, cond_mtime)
|
||||
if cond_resp:
|
||||
return cond_resp
|
||||
|
||||
if request.method == "GET":
|
||||
range_header = request.headers.get("Range")
|
||||
|
||||
@@ -2780,7 +3009,7 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
try:
|
||||
stat = path.stat()
|
||||
file_size = stat.st_size
|
||||
etag = storage._compute_etag(path)
|
||||
etag = metadata.get("__etag__") or storage._compute_etag(path)
|
||||
except PermissionError:
|
||||
return _error_response("AccessDenied", "Permission denied accessing object", 403)
|
||||
except OSError as exc:
|
||||
@@ -2801,7 +3030,7 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
f.seek(start_pos)
|
||||
remaining = length_to_read
|
||||
while remaining > 0:
|
||||
chunk_size = min(65536, remaining)
|
||||
chunk_size = min(262144, remaining)
|
||||
chunk = f.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
@@ -2828,7 +3057,7 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
try:
|
||||
stat = path.stat()
|
||||
response = Response(status=200)
|
||||
etag = storage._compute_etag(path)
|
||||
etag = metadata.get("__etag__") or storage._compute_etag(path)
|
||||
except PermissionError:
|
||||
return _error_response("AccessDenied", "Permission denied accessing object", 403)
|
||||
except OSError as exc:
|
||||
@@ -2836,10 +3065,7 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
response.headers["Content-Type"] = mimetype
|
||||
logged_bytes = 0
|
||||
|
||||
try:
|
||||
file_stat = path.stat() if not is_encrypted else None
|
||||
except (PermissionError, OSError):
|
||||
file_stat = None
|
||||
file_stat = stat if not is_encrypted else None
|
||||
_apply_object_headers(response, file_stat=file_stat, metadata=metadata, etag=etag)
|
||||
|
||||
if request.method == "GET":
|
||||
@@ -2856,6 +3082,7 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
if value:
|
||||
response.headers[header] = _sanitize_header_value(value)
|
||||
|
||||
if current_app.logger.isEnabledFor(logging.INFO):
|
||||
action = "Object read" if request.method == "GET" else "Object head"
|
||||
current_app.logger.info(action, extra={"bucket": bucket_name, "key": object_key, "bytes": logged_bytes})
|
||||
return response
|
||||
@@ -2875,6 +3102,7 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
|
||||
storage.delete_object(bucket_name, object_key)
|
||||
lock_service.delete_object_lock_metadata(bucket_name, object_key)
|
||||
if current_app.logger.isEnabledFor(logging.INFO):
|
||||
current_app.logger.info("Object deleted", extra={"bucket": bucket_name, "key": object_key})
|
||||
|
||||
principal, _ = _require_principal()
|
||||
@@ -2963,7 +3191,11 @@ def _bucket_policy_handler(bucket_name: str) -> Response:
|
||||
store.delete_policy(bucket_name)
|
||||
current_app.logger.info("Bucket policy removed", extra={"bucket": bucket_name})
|
||||
return Response(status=204)
|
||||
payload = request.get_json(silent=True)
|
||||
raw_body = request.get_data(cache=False) or b""
|
||||
try:
|
||||
payload = json.loads(raw_body)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return _error_response("MalformedPolicy", "Policy document must be JSON", 400)
|
||||
if not payload:
|
||||
return _error_response("MalformedPolicy", "Policy document must be JSON", 400)
|
||||
try:
|
||||
@@ -2974,6 +3206,32 @@ def _bucket_policy_handler(bucket_name: str) -> Response:
|
||||
return Response(status=204)
|
||||
|
||||
|
||||
def _bucket_policy_status_handler(bucket_name: str) -> Response:
|
||||
if request.method != "GET":
|
||||
return _method_not_allowed(["GET"])
|
||||
principal, error = _require_principal()
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
storage = _storage()
|
||||
if not storage.bucket_exists(bucket_name):
|
||||
return _error_response("NoSuchBucket", "Bucket does not exist", 404)
|
||||
store = _bucket_policies()
|
||||
policy = store.get_policy(bucket_name)
|
||||
is_public = False
|
||||
if policy:
|
||||
for statement in policy.get("Statement", []):
|
||||
if statement.get("Effect") == "Allow" and statement.get("Principal") == "*":
|
||||
is_public = True
|
||||
break
|
||||
root = Element("PolicyStatus")
|
||||
SubElement(root, "IsPublic").text = "TRUE" if is_public else "FALSE"
|
||||
return _xml_response(root)
|
||||
|
||||
|
||||
def _bucket_replication_handler(bucket_name: str) -> Response:
|
||||
if request.method not in {"GET", "PUT", "DELETE"}:
|
||||
return _method_not_allowed(["GET", "PUT", "DELETE"])
|
||||
@@ -2981,7 +3239,7 @@ def _bucket_replication_handler(bucket_name: str) -> Response:
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "policy")
|
||||
_authorize_action(principal, bucket_name, "replication")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
storage = _storage()
|
||||
@@ -3055,6 +3313,79 @@ def _parse_replication_config(bucket_name: str, payload: bytes):
|
||||
)
|
||||
|
||||
|
||||
def _bucket_website_handler(bucket_name: str) -> Response:
|
||||
if request.method not in {"GET", "PUT", "DELETE"}:
|
||||
return _method_not_allowed(["GET", "PUT", "DELETE"])
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _error_response("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
principal, error = _require_principal()
|
||||
if error:
|
||||
return error
|
||||
try:
|
||||
_authorize_action(principal, bucket_name, "website")
|
||||
except IamError as exc:
|
||||
return _error_response("AccessDenied", str(exc), 403)
|
||||
storage = _storage()
|
||||
if request.method == "GET":
|
||||
try:
|
||||
config = storage.get_bucket_website(bucket_name)
|
||||
except StorageError as exc:
|
||||
return _error_response("NoSuchBucket", str(exc), 404)
|
||||
if not config:
|
||||
return _error_response("NoSuchWebsiteConfiguration", "The specified bucket does not have a website configuration", 404)
|
||||
root = Element("WebsiteConfiguration")
|
||||
root.set("xmlns", S3_NS)
|
||||
index_doc = config.get("index_document")
|
||||
if index_doc:
|
||||
idx_el = SubElement(root, "IndexDocument")
|
||||
SubElement(idx_el, "Suffix").text = index_doc
|
||||
error_doc = config.get("error_document")
|
||||
if error_doc:
|
||||
err_el = SubElement(root, "ErrorDocument")
|
||||
SubElement(err_el, "Key").text = error_doc
|
||||
return _xml_response(root)
|
||||
if request.method == "DELETE":
|
||||
try:
|
||||
storage.set_bucket_website(bucket_name, None)
|
||||
except StorageError as exc:
|
||||
return _error_response("NoSuchBucket", str(exc), 404)
|
||||
current_app.logger.info("Bucket website config deleted", extra={"bucket": bucket_name})
|
||||
return Response(status=204)
|
||||
ct_error = _require_xml_content_type()
|
||||
if ct_error:
|
||||
return ct_error
|
||||
payload = request.get_data(cache=False) or b""
|
||||
if not payload.strip():
|
||||
return _error_response("MalformedXML", "Request body is required", 400)
|
||||
try:
|
||||
root = _parse_xml_with_limit(payload)
|
||||
except ParseError:
|
||||
return _error_response("MalformedXML", "Unable to parse XML document", 400)
|
||||
if _strip_ns(root.tag) != "WebsiteConfiguration":
|
||||
return _error_response("MalformedXML", "Root element must be WebsiteConfiguration", 400)
|
||||
index_el = _find_element(root, "IndexDocument")
|
||||
if index_el is None:
|
||||
return _error_response("InvalidArgument", "IndexDocument is required", 400)
|
||||
suffix_el = _find_element(index_el, "Suffix")
|
||||
if suffix_el is None or not (suffix_el.text or "").strip():
|
||||
return _error_response("InvalidArgument", "IndexDocument Suffix is required", 400)
|
||||
index_suffix = suffix_el.text.strip()
|
||||
if "/" in index_suffix:
|
||||
return _error_response("InvalidArgument", "IndexDocument Suffix must not contain '/'", 400)
|
||||
website_config: Dict[str, Any] = {"index_document": index_suffix}
|
||||
error_el = _find_element(root, "ErrorDocument")
|
||||
if error_el is not None:
|
||||
key_el = _find_element(error_el, "Key")
|
||||
if key_el is not None and (key_el.text or "").strip():
|
||||
website_config["error_document"] = key_el.text.strip()
|
||||
try:
|
||||
storage.set_bucket_website(bucket_name, website_config)
|
||||
except StorageError as exc:
|
||||
return _error_response("NoSuchBucket", str(exc), 404)
|
||||
current_app.logger.info("Bucket website config updated", extra={"bucket": bucket_name, "index": index_suffix})
|
||||
return Response(status=200)
|
||||
|
||||
|
||||
def _parse_destination_arn(arn: str) -> tuple:
|
||||
if not arn.startswith("arn:aws:s3:::"):
|
||||
raise ValueError(f"Invalid ARN format: {arn}")
|
||||
@@ -3113,12 +3444,30 @@ def head_object(bucket_name: str, object_key: str) -> Response:
|
||||
_authorize_action(principal, bucket_name, "read", object_key=object_key)
|
||||
path = _storage().get_object_path(bucket_name, object_key)
|
||||
metadata = _storage().get_object_metadata(bucket_name, object_key)
|
||||
stat = path.stat()
|
||||
etag = _storage()._compute_etag(path)
|
||||
etag = metadata.get("__etag__") or _storage()._compute_etag(path)
|
||||
|
||||
head_mtime = float(metadata["__last_modified__"]) if "__last_modified__" in metadata else None
|
||||
if head_mtime is None:
|
||||
try:
|
||||
head_mtime = path.stat().st_mtime
|
||||
except OSError:
|
||||
pass
|
||||
cond_resp = _check_conditional_headers(etag, head_mtime)
|
||||
if cond_resp:
|
||||
return cond_resp
|
||||
|
||||
cached_size = metadata.get("__size__")
|
||||
cached_mtime = metadata.get("__last_modified__")
|
||||
if cached_size is not None and cached_mtime is not None:
|
||||
size_val = int(cached_size)
|
||||
mtime_val = float(cached_mtime)
|
||||
response = Response(status=200)
|
||||
_apply_object_headers(response, file_stat=None, metadata=metadata, etag=etag, size_override=size_val, mtime_override=mtime_val)
|
||||
else:
|
||||
stat = path.stat()
|
||||
response = Response(status=200)
|
||||
_apply_object_headers(response, file_stat=stat, metadata=metadata, etag=etag)
|
||||
response.headers["Content-Type"] = mimetypes.guess_type(object_key)[0] or "application/octet-stream"
|
||||
response.headers["Content-Type"] = metadata.get("__content_type__") or mimetypes.guess_type(object_key)[0] or "application/octet-stream"
|
||||
return response
|
||||
except (StorageError, FileNotFoundError):
|
||||
return _error_response("NoSuchKey", "Object not found", 404)
|
||||
@@ -3207,7 +3556,7 @@ def _copy_object(dest_bucket: str, dest_key: str, copy_source: str) -> Response:
|
||||
if validation_error:
|
||||
return _error_response("InvalidArgument", validation_error, 400)
|
||||
else:
|
||||
metadata = source_metadata
|
||||
metadata = {k: v for k, v in source_metadata.items() if not (k.startswith("__") and k.endswith("__"))}
|
||||
|
||||
try:
|
||||
with source_path.open("rb") as stream:
|
||||
@@ -3348,6 +3697,8 @@ def _initiate_multipart_upload(bucket_name: str, object_key: str) -> Response:
|
||||
return error
|
||||
|
||||
metadata = _extract_request_metadata()
|
||||
content_type = request.headers.get("Content-Type")
|
||||
metadata["__content_type__"] = content_type or mimetypes.guess_type(object_key)[0] or "application/octet-stream"
|
||||
try:
|
||||
upload_id = _storage().initiate_multipart_upload(
|
||||
bucket_name,
|
||||
@@ -3400,6 +3751,15 @@ def _upload_part(bucket_name: str, object_key: str) -> Response:
|
||||
return _error_response("NoSuchUpload", str(exc), 404)
|
||||
return _error_response("InvalidArgument", str(exc), 400)
|
||||
|
||||
content_md5 = request.headers.get("Content-MD5")
|
||||
if content_md5 and etag:
|
||||
try:
|
||||
expected_md5 = base64.b64decode(content_md5).hex()
|
||||
except Exception:
|
||||
return _error_response("InvalidDigest", "Content-MD5 header is not valid base64", 400)
|
||||
if expected_md5 != etag:
|
||||
return _error_response("BadDigest", "The Content-MD5 you specified did not match what we received", 400)
|
||||
|
||||
response = Response(status=200)
|
||||
response.headers["ETag"] = f'"{etag}"'
|
||||
return response
|
||||
|
||||
296
app/s3_client.py
Normal file
296
app/s3_client.py
Normal file
@@ -0,0 +1,296 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Generator, Optional
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError, EndpointConnectionError, ConnectionClosedError
|
||||
from flask import current_app, session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
UI_PROXY_USER_AGENT = "MyFSIO-UIProxy/1.0"
|
||||
|
||||
_BOTO_ERROR_MAP = {
|
||||
"NoSuchBucket": 404,
|
||||
"NoSuchKey": 404,
|
||||
"NoSuchUpload": 404,
|
||||
"BucketAlreadyExists": 409,
|
||||
"BucketAlreadyOwnedByYou": 409,
|
||||
"BucketNotEmpty": 409,
|
||||
"AccessDenied": 403,
|
||||
"InvalidAccessKeyId": 403,
|
||||
"SignatureDoesNotMatch": 403,
|
||||
"InvalidBucketName": 400,
|
||||
"InvalidArgument": 400,
|
||||
"MalformedXML": 400,
|
||||
"EntityTooLarge": 400,
|
||||
"QuotaExceeded": 403,
|
||||
}
|
||||
|
||||
_UPLOAD_REGISTRY_MAX_AGE = 86400
|
||||
_UPLOAD_REGISTRY_CLEANUP_INTERVAL = 3600
|
||||
|
||||
|
||||
class UploadRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._entries: dict[str, tuple[str, str, float]] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._last_cleanup = time.monotonic()
|
||||
|
||||
def register(self, upload_id: str, bucket_name: str, object_key: str) -> None:
|
||||
with self._lock:
|
||||
self._entries[upload_id] = (bucket_name, object_key, time.monotonic())
|
||||
self._maybe_cleanup()
|
||||
|
||||
def get_key(self, upload_id: str, bucket_name: str) -> Optional[str]:
|
||||
with self._lock:
|
||||
entry = self._entries.get(upload_id)
|
||||
if entry is None:
|
||||
return None
|
||||
stored_bucket, key, created_at = entry
|
||||
if stored_bucket != bucket_name:
|
||||
return None
|
||||
if time.monotonic() - created_at > _UPLOAD_REGISTRY_MAX_AGE:
|
||||
del self._entries[upload_id]
|
||||
return None
|
||||
return key
|
||||
|
||||
def remove(self, upload_id: str) -> None:
|
||||
with self._lock:
|
||||
self._entries.pop(upload_id, None)
|
||||
|
||||
def _maybe_cleanup(self) -> None:
|
||||
now = time.monotonic()
|
||||
if now - self._last_cleanup < _UPLOAD_REGISTRY_CLEANUP_INTERVAL:
|
||||
return
|
||||
self._last_cleanup = now
|
||||
cutoff = now - _UPLOAD_REGISTRY_MAX_AGE
|
||||
stale = [uid for uid, (_, _, ts) in self._entries.items() if ts < cutoff]
|
||||
for uid in stale:
|
||||
del self._entries[uid]
|
||||
|
||||
|
||||
class S3ProxyClient:
|
||||
def __init__(self, api_base_url: str, region: str = "us-east-1") -> None:
|
||||
if not api_base_url:
|
||||
raise ValueError("api_base_url is required for S3ProxyClient")
|
||||
self._api_base_url = api_base_url.rstrip("/")
|
||||
self._region = region
|
||||
self.upload_registry = UploadRegistry()
|
||||
|
||||
@property
|
||||
def api_base_url(self) -> str:
|
||||
return self._api_base_url
|
||||
|
||||
def get_client(self, access_key: str, secret_key: str) -> Any:
|
||||
if not access_key or not secret_key:
|
||||
raise ValueError("Both access_key and secret_key are required")
|
||||
config = Config(
|
||||
user_agent_extra=UI_PROXY_USER_AGENT,
|
||||
connect_timeout=5,
|
||||
read_timeout=30,
|
||||
retries={"max_attempts": 0},
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
)
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=self._api_base_url,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=self._region,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def _get_proxy() -> S3ProxyClient:
|
||||
proxy = current_app.extensions.get("s3_proxy")
|
||||
if proxy is None:
|
||||
raise RuntimeError(
|
||||
"S3 proxy not configured. Set API_BASE_URL or run both API and UI servers."
|
||||
)
|
||||
return proxy
|
||||
|
||||
|
||||
def _get_session_creds() -> tuple[str, str]:
|
||||
secret_store = current_app.extensions["secret_store"]
|
||||
secret_store.purge_expired()
|
||||
token = session.get("cred_token")
|
||||
if not token:
|
||||
raise PermissionError("Not authenticated")
|
||||
creds = secret_store.peek(token)
|
||||
if not creds:
|
||||
raise PermissionError("Session expired")
|
||||
access_key = creds.get("access_key", "")
|
||||
secret_key = creds.get("secret_key", "")
|
||||
if not access_key or not secret_key:
|
||||
raise PermissionError("Invalid session credentials")
|
||||
return access_key, secret_key
|
||||
|
||||
|
||||
def get_session_s3_client() -> Any:
|
||||
proxy = _get_proxy()
|
||||
access_key, secret_key = _get_session_creds()
|
||||
return proxy.get_client(access_key, secret_key)
|
||||
|
||||
|
||||
def get_upload_registry() -> UploadRegistry:
|
||||
return _get_proxy().upload_registry
|
||||
|
||||
|
||||
def handle_client_error(exc: ClientError) -> tuple[dict[str, str], int]:
|
||||
error_info = exc.response.get("Error", {})
|
||||
code = error_info.get("Code", "InternalError")
|
||||
message = error_info.get("Message") or "S3 operation failed"
|
||||
http_status = _BOTO_ERROR_MAP.get(code)
|
||||
if http_status is None:
|
||||
http_status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 500)
|
||||
return {"error": message}, http_status
|
||||
|
||||
|
||||
def handle_connection_error(exc: Exception) -> tuple[dict[str, str], int]:
|
||||
logger.error("S3 API connection failed: %s", exc)
|
||||
return {"error": "S3 API server is unreachable. Ensure the API server is running."}, 502
|
||||
|
||||
|
||||
def format_datetime_display(dt: Any, display_tz: str = "UTC") -> str:
|
||||
from .ui import _format_datetime_display
|
||||
return _format_datetime_display(dt, display_tz)
|
||||
|
||||
|
||||
def format_datetime_iso(dt: Any, display_tz: str = "UTC") -> str:
|
||||
from .ui import _format_datetime_iso
|
||||
return _format_datetime_iso(dt, display_tz)
|
||||
|
||||
|
||||
def build_url_templates(bucket_name: str) -> dict[str, str]:
|
||||
from flask import url_for
|
||||
preview_t = url_for("ui.object_preview", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
delete_t = url_for("ui.delete_object", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
presign_t = url_for("ui.object_presign", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
versions_t = url_for("ui.object_versions", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
restore_t = url_for(
|
||||
"ui.restore_object_version",
|
||||
bucket_name=bucket_name,
|
||||
object_key="KEY_PLACEHOLDER",
|
||||
version_id="VERSION_ID_PLACEHOLDER",
|
||||
)
|
||||
tags_t = url_for("ui.object_tags", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
copy_t = url_for("ui.copy_object", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
move_t = url_for("ui.move_object", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
metadata_t = url_for("ui.object_metadata", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
return {
|
||||
"preview": preview_t,
|
||||
"download": preview_t + "?download=1",
|
||||
"presign": presign_t,
|
||||
"delete": delete_t,
|
||||
"versions": versions_t,
|
||||
"restore": restore_t,
|
||||
"tags": tags_t,
|
||||
"copy": copy_t,
|
||||
"move": move_t,
|
||||
"metadata": metadata_t,
|
||||
}
|
||||
|
||||
|
||||
def translate_list_objects(
|
||||
boto3_response: dict[str, Any],
|
||||
url_templates: dict[str, str],
|
||||
display_tz: str = "UTC",
|
||||
versioning_enabled: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
objects_data = []
|
||||
for obj in boto3_response.get("Contents", []):
|
||||
last_mod = obj["LastModified"]
|
||||
objects_data.append({
|
||||
"key": obj["Key"],
|
||||
"size": obj["Size"],
|
||||
"last_modified": last_mod.isoformat(),
|
||||
"last_modified_display": format_datetime_display(last_mod, display_tz),
|
||||
"last_modified_iso": format_datetime_iso(last_mod, display_tz),
|
||||
"etag": obj.get("ETag", "").strip('"'),
|
||||
})
|
||||
return {
|
||||
"objects": objects_data,
|
||||
"is_truncated": boto3_response.get("IsTruncated", False),
|
||||
"next_continuation_token": boto3_response.get("NextContinuationToken"),
|
||||
"total_count": boto3_response.get("KeyCount", len(objects_data)),
|
||||
"versioning_enabled": versioning_enabled,
|
||||
"url_templates": url_templates,
|
||||
}
|
||||
|
||||
|
||||
def get_versioning_via_s3(client: Any, bucket_name: str) -> bool:
|
||||
try:
|
||||
resp = client.get_bucket_versioning(Bucket=bucket_name)
|
||||
return resp.get("Status") == "Enabled"
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code != "NoSuchBucket":
|
||||
logger.warning("Failed to check versioning for %s: %s", bucket_name, code)
|
||||
return False
|
||||
|
||||
|
||||
def stream_objects_ndjson(
|
||||
client: Any,
|
||||
bucket_name: str,
|
||||
prefix: Optional[str],
|
||||
url_templates: dict[str, str],
|
||||
display_tz: str = "UTC",
|
||||
versioning_enabled: bool = False,
|
||||
delimiter: Optional[str] = None,
|
||||
) -> Generator[str, None, None]:
|
||||
meta_line = json.dumps({
|
||||
"type": "meta",
|
||||
"versioning_enabled": versioning_enabled,
|
||||
"url_templates": url_templates,
|
||||
}) + "\n"
|
||||
yield meta_line
|
||||
|
||||
yield json.dumps({"type": "count", "total_count": 0}) + "\n"
|
||||
|
||||
kwargs: dict[str, Any] = {"Bucket": bucket_name, "MaxKeys": 1000}
|
||||
if prefix:
|
||||
kwargs["Prefix"] = prefix
|
||||
if delimiter:
|
||||
kwargs["Delimiter"] = delimiter
|
||||
|
||||
running_count = 0
|
||||
try:
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(**kwargs):
|
||||
for cp in page.get("CommonPrefixes", []):
|
||||
yield json.dumps({
|
||||
"type": "folder",
|
||||
"prefix": cp["Prefix"],
|
||||
}) + "\n"
|
||||
page_contents = page.get("Contents", [])
|
||||
for obj in page_contents:
|
||||
last_mod = obj["LastModified"]
|
||||
yield json.dumps({
|
||||
"type": "object",
|
||||
"key": obj["Key"],
|
||||
"size": obj["Size"],
|
||||
"last_modified": last_mod.isoformat(),
|
||||
"last_modified_display": format_datetime_display(last_mod, display_tz),
|
||||
"last_modified_iso": format_datetime_iso(last_mod, display_tz),
|
||||
"etag": obj.get("ETag", "").strip('"'),
|
||||
}) + "\n"
|
||||
running_count += len(page_contents)
|
||||
yield json.dumps({"type": "count", "total_count": running_count}) + "\n"
|
||||
except ClientError as exc:
|
||||
error_msg = exc.response.get("Error", {}).get("Message", "S3 operation failed")
|
||||
yield json.dumps({"type": "error", "error": error_msg}) + "\n"
|
||||
return
|
||||
except (EndpointConnectionError, ConnectionClosedError):
|
||||
yield json.dumps({"type": "error", "error": "S3 API server is unreachable"}) + "\n"
|
||||
return
|
||||
|
||||
yield json.dumps({"type": "done"}) + "\n"
|
||||
1062
app/storage.py
1062
app/storage.py
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
APP_VERSION = "0.2.7"
|
||||
APP_VERSION = "0.4.1"
|
||||
|
||||
|
||||
def get_version() -> str:
|
||||
|
||||
108
app/website_domains.py
Normal file
108
app/website_domains.py
Normal file
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
_DOMAIN_RE = re.compile(
|
||||
r"^(?!-)[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$"
|
||||
)
|
||||
|
||||
|
||||
def normalize_domain(raw: str) -> str:
|
||||
raw = raw.strip().lower()
|
||||
for prefix in ("https://", "http://"):
|
||||
if raw.startswith(prefix):
|
||||
raw = raw[len(prefix):]
|
||||
raw = raw.split("/", 1)[0]
|
||||
raw = raw.split("?", 1)[0]
|
||||
raw = raw.split("#", 1)[0]
|
||||
if ":" in raw:
|
||||
raw = raw.rsplit(":", 1)[0]
|
||||
return raw
|
||||
|
||||
|
||||
def is_valid_domain(domain: str) -> bool:
|
||||
if not domain or len(domain) > 253:
|
||||
return False
|
||||
return bool(_DOMAIN_RE.match(domain))
|
||||
|
||||
|
||||
class WebsiteDomainStore:
|
||||
def __init__(self, config_path: Path) -> None:
|
||||
self.config_path = config_path
|
||||
self._lock = threading.Lock()
|
||||
self._domains: Dict[str, str] = {}
|
||||
self._last_mtime: float = 0.0
|
||||
self.reload()
|
||||
|
||||
def reload(self) -> None:
|
||||
if not self.config_path.exists():
|
||||
self._domains = {}
|
||||
self._last_mtime = 0.0
|
||||
return
|
||||
try:
|
||||
self._last_mtime = self.config_path.stat().st_mtime
|
||||
with open(self.config_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict):
|
||||
self._domains = {k.lower(): v for k, v in data.items()}
|
||||
else:
|
||||
self._domains = {}
|
||||
except (OSError, json.JSONDecodeError):
|
||||
self._domains = {}
|
||||
|
||||
def _maybe_reload(self) -> None:
|
||||
try:
|
||||
if self.config_path.exists():
|
||||
mtime = self.config_path.stat().st_mtime
|
||||
if mtime != self._last_mtime:
|
||||
self._last_mtime = mtime
|
||||
with open(self.config_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict):
|
||||
self._domains = {k.lower(): v for k, v in data.items()}
|
||||
else:
|
||||
self._domains = {}
|
||||
elif self._domains:
|
||||
self._domains = {}
|
||||
self._last_mtime = 0.0
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
|
||||
def _save(self) -> None:
|
||||
self.config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.config_path, "w", encoding="utf-8") as f:
|
||||
json.dump(self._domains, f, indent=2)
|
||||
self._last_mtime = self.config_path.stat().st_mtime
|
||||
|
||||
def list_all(self) -> List[Dict[str, str]]:
|
||||
with self._lock:
|
||||
self._maybe_reload()
|
||||
return [{"domain": d, "bucket": b} for d, b in self._domains.items()]
|
||||
|
||||
def get_bucket(self, domain: str) -> Optional[str]:
|
||||
with self._lock:
|
||||
self._maybe_reload()
|
||||
return self._domains.get(domain.lower())
|
||||
|
||||
def get_domains_for_bucket(self, bucket: str) -> List[str]:
|
||||
with self._lock:
|
||||
self._maybe_reload()
|
||||
return [d for d, b in self._domains.items() if b == bucket]
|
||||
|
||||
def set_mapping(self, domain: str, bucket: str) -> None:
|
||||
with self._lock:
|
||||
self._domains[domain.lower()] = bucket
|
||||
self._save()
|
||||
|
||||
def delete_mapping(self, domain: str) -> bool:
|
||||
with self._lock:
|
||||
key = domain.lower()
|
||||
if key not in self._domains:
|
||||
return False
|
||||
del self._domains[key]
|
||||
self._save()
|
||||
return True
|
||||
422
docs.md
422
docs.md
@@ -7,7 +7,7 @@ This document expands on the README to describe the full workflow for running, c
|
||||
MyFSIO ships two Flask entrypoints that share the same storage, IAM, and bucket-policy state:
|
||||
|
||||
- **API server** – Implements the S3-compatible REST API, policy evaluation, and Signature Version 4 presign service.
|
||||
- **UI server** – Provides the browser console for buckets, IAM, and policies. It proxies to the API for presign operations.
|
||||
- **UI server** – Provides the browser console for buckets, IAM, and policies. It proxies all storage operations through the S3 API via boto3 (SigV4-signed), mirroring the architecture used by MinIO and Garage.
|
||||
|
||||
Both servers read `AppConfig`, so editing JSON stores on disk instantly affects both surfaces.
|
||||
|
||||
@@ -136,21 +136,24 @@ All configuration is done via environment variables. The table below lists every
|
||||
| `MAX_UPLOAD_SIZE` | `1073741824` (1 GiB) | Bytes. Caps incoming uploads in both API + UI. |
|
||||
| `UI_PAGE_SIZE` | `100` | `MaxKeys` hint shown in listings. |
|
||||
| `SECRET_KEY` | Auto-generated | Flask session key. Auto-generates and persists if not set. **Set explicitly in production.** |
|
||||
| `API_BASE_URL` | `None` | Public URL for presigned URLs. Required behind proxies. |
|
||||
| `API_BASE_URL` | `http://127.0.0.1:5000` | Internal S3 API URL used by the web UI proxy. Also used for presigned URL generation. Set to your public URL if running behind a reverse proxy. |
|
||||
| `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. |
|
||||
| `AWS_SERVICE` | `s3` | Service string for SigV4. |
|
||||
| `DISPLAY_TIMEZONE` | `UTC` | Timezone for timestamps in the web UI (e.g., `US/Eastern`, `Asia/Tokyo`). |
|
||||
|
||||
### IAM & Security
|
||||
|
||||
| Variable | Default | Notes |
|
||||
| --- | --- | --- |
|
||||
| `IAM_CONFIG` | `data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. |
|
||||
| `IAM_CONFIG` | `data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. Encrypted at rest when `SECRET_KEY` is set. |
|
||||
| `BUCKET_POLICY_PATH` | `data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). |
|
||||
| `AUTH_MAX_ATTEMPTS` | `5` | Failed login attempts before lockout. |
|
||||
| `AUTH_LOCKOUT_MINUTES` | `15` | Lockout duration after max failed attempts. |
|
||||
| `SESSION_LIFETIME_DAYS` | `30` | How long UI sessions remain valid. |
|
||||
| `SECRET_TTL_SECONDS` | `300` | TTL for ephemeral secrets (presigned URLs). |
|
||||
| `UI_ENFORCE_BUCKET_POLICIES` | `false` | Whether the UI should enforce bucket policies. |
|
||||
| `ADMIN_ACCESS_KEY` | (none) | Custom access key for the admin user on first run or credential reset. If unset, a random key is generated. |
|
||||
| `ADMIN_SECRET_KEY` | (none) | Custom secret key for the admin user on first run or credential reset. If unset, a random key is generated. |
|
||||
|
||||
### CORS (Cross-Origin Resource Sharing)
|
||||
|
||||
@@ -170,15 +173,16 @@ All configuration is done via environment variables. The table below lists every
|
||||
| `RATE_LIMIT_BUCKET_OPS` | `120 per minute` | Rate limit for bucket operations (PUT/DELETE/GET/POST on `/<bucket>`). |
|
||||
| `RATE_LIMIT_OBJECT_OPS` | `240 per minute` | Rate limit for object operations (PUT/GET/DELETE/POST on `/<bucket>/<key>`). |
|
||||
| `RATE_LIMIT_HEAD_OPS` | `100 per minute` | Rate limit for HEAD requests (bucket and object). |
|
||||
| `RATE_LIMIT_ADMIN` | `60 per minute` | Rate limit for admin API endpoints (`/admin/*`). |
|
||||
| `RATE_LIMIT_STORAGE_URI` | `memory://` | Storage backend for rate limits. Use `redis://host:port` for distributed setups. |
|
||||
|
||||
### Server Configuration
|
||||
|
||||
| Variable | Default | Notes |
|
||||
| --- | --- | --- |
|
||||
| `SERVER_THREADS` | `0` (auto) | Waitress worker threads (1-64). Set to `0` for auto-calculation based on CPU cores (×2). |
|
||||
| `SERVER_CONNECTION_LIMIT` | `0` (auto) | Maximum concurrent connections (10-1000). Set to `0` for auto-calculation based on available RAM. |
|
||||
| `SERVER_BACKLOG` | `0` (auto) | TCP listen backlog (64-4096). Set to `0` for auto-calculation (connection_limit × 2). |
|
||||
| `SERVER_THREADS` | `0` (auto) | Granian blocking threads (1-64). Set to `0` for auto-calculation based on CPU cores (×2). |
|
||||
| `SERVER_CONNECTION_LIMIT` | `0` (auto) | Maximum concurrent requests per worker (10-1000). Set to `0` for auto-calculation based on available RAM. |
|
||||
| `SERVER_BACKLOG` | `0` (auto) | TCP listen backlog (128-4096). Set to `0` for auto-calculation (connection_limit × 2). |
|
||||
| `SERVER_CHANNEL_TIMEOUT` | `120` | Seconds before idle connections are closed (10-300). |
|
||||
|
||||
### Logging
|
||||
@@ -248,6 +252,60 @@ Once enabled, configure lifecycle rules via:
|
||||
</LifecycleConfiguration>
|
||||
```
|
||||
|
||||
## Garbage Collection
|
||||
|
||||
The garbage collector (GC) automatically cleans up orphaned data that accumulates over time: stale temporary files from failed uploads, abandoned multipart uploads, stale lock files, orphaned metadata entries, orphaned version files, and empty directories.
|
||||
|
||||
### Enabling GC
|
||||
|
||||
By default, GC is disabled. Enable it by setting:
|
||||
|
||||
```bash
|
||||
GC_ENABLED=true python run.py
|
||||
```
|
||||
|
||||
Or in your `myfsio.env` file:
|
||||
```
|
||||
GC_ENABLED=true
|
||||
GC_INTERVAL_HOURS=6 # Run every 6 hours (default)
|
||||
GC_TEMP_FILE_MAX_AGE_HOURS=24 # Delete temp files older than 24h
|
||||
GC_MULTIPART_MAX_AGE_DAYS=7 # Delete orphaned multipart uploads older than 7 days
|
||||
GC_LOCK_FILE_MAX_AGE_HOURS=1 # Delete stale lock files older than 1h
|
||||
GC_DRY_RUN=false # Set to true to log without deleting
|
||||
```
|
||||
|
||||
### What Gets Cleaned
|
||||
|
||||
| Type | Location | Condition |
|
||||
|------|----------|-----------|
|
||||
| **Temp files** | `.myfsio.sys/tmp/` | Older than `GC_TEMP_FILE_MAX_AGE_HOURS` |
|
||||
| **Orphaned multipart uploads** | `.myfsio.sys/multipart/` and `<bucket>/.multipart/` | Older than `GC_MULTIPART_MAX_AGE_DAYS` |
|
||||
| **Stale lock files** | `.myfsio.sys/buckets/<bucket>/locks/` | Older than `GC_LOCK_FILE_MAX_AGE_HOURS` |
|
||||
| **Orphaned metadata** | `.myfsio.sys/buckets/<bucket>/meta/` and `<bucket>/.meta/` | Object file no longer exists |
|
||||
| **Orphaned versions** | `.myfsio.sys/buckets/<bucket>/versions/` and `<bucket>/.versions/` | Main object no longer exists |
|
||||
| **Empty directories** | Various internal directories | Directory is empty after cleanup |
|
||||
|
||||
### Admin API
|
||||
|
||||
All GC endpoints require admin (`iam:*`) permissions.
|
||||
|
||||
| Method | Route | Description |
|
||||
|--------|-------|-------------|
|
||||
| `GET` | `/admin/gc/status` | Get GC status and configuration |
|
||||
| `POST` | `/admin/gc/run` | Trigger a manual GC run (body: `{"dry_run": true}` for preview) |
|
||||
| `GET` | `/admin/gc/history` | Get GC execution history (query: `?limit=50&offset=0`) |
|
||||
|
||||
### Dry Run Mode
|
||||
|
||||
Set `GC_DRY_RUN=true` to log what would be deleted without actually removing anything. You can also trigger a one-time dry run via the admin API:
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:5000/admin/gc/run" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"dry_run": true}'
|
||||
```
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
| Variable | Default | Notes |
|
||||
@@ -256,6 +314,12 @@ Once enabled, configure lifecycle rules via:
|
||||
| `MULTIPART_MIN_PART_SIZE` | `5242880` (5 MB) | Minimum part size for multipart uploads. |
|
||||
| `BUCKET_STATS_CACHE_TTL` | `60` | Seconds to cache bucket statistics. |
|
||||
| `BULK_DELETE_MAX_KEYS` | `500` | Maximum keys per bulk delete request. |
|
||||
| `BULK_DOWNLOAD_MAX_BYTES` | `1073741824` (1 GiB) | Maximum total size for bulk ZIP downloads. |
|
||||
| `OBJECT_CACHE_TTL` | `60` | Seconds to cache object metadata. |
|
||||
|
||||
#### Gzip Compression
|
||||
|
||||
API responses for JSON, XML, HTML, CSS, and JavaScript are automatically gzip-compressed when the client sends `Accept-Encoding: gzip`. Compression activates for responses larger than 500 bytes and is handled by a WSGI middleware (`app/compression.py`). Binary object downloads and streaming responses are never compressed. No configuration is needed.
|
||||
|
||||
### Server Settings
|
||||
|
||||
@@ -269,13 +333,14 @@ Once enabled, configure lifecycle rules via:
|
||||
|
||||
Before deploying to production, ensure you:
|
||||
|
||||
1. **Set `SECRET_KEY`** - Use a strong, unique value (e.g., `openssl rand -base64 32`)
|
||||
1. **Set `SECRET_KEY`** - Use a strong, unique value (e.g., `openssl rand -base64 32`). This also enables IAM config encryption at rest.
|
||||
2. **Restrict CORS** - Set `CORS_ORIGINS` to your specific domains instead of `*`
|
||||
3. **Configure `API_BASE_URL`** - Required for correct presigned URLs behind proxies
|
||||
4. **Enable HTTPS** - Use a reverse proxy (nginx, Cloudflare) with TLS termination
|
||||
5. **Review rate limits** - Adjust `RATE_LIMIT_DEFAULT` based on your needs
|
||||
6. **Secure master keys** - Back up `ENCRYPTION_MASTER_KEY_PATH` if using encryption
|
||||
7. **Use `--prod` flag** - Runs with Waitress instead of Flask dev server
|
||||
7. **Use `--prod` flag** - Runs with Granian instead of Flask dev server
|
||||
8. **Set credential expiry** - Assign `expires_at` to non-admin users for time-limited access
|
||||
|
||||
### Proxy Configuration
|
||||
|
||||
@@ -285,6 +350,75 @@ If running behind a reverse proxy (e.g., Nginx, Cloudflare, or a tunnel), ensure
|
||||
|
||||
The application automatically trusts these headers to generate correct presigned URLs (e.g., `https://s3.example.com/...` instead of `http://127.0.0.1:5000/...`). Alternatively, you can explicitly set `API_BASE_URL` to your public endpoint.
|
||||
|
||||
| Variable | Default | Notes |
|
||||
| --- | --- | --- |
|
||||
| `NUM_TRUSTED_PROXIES` | `1` | Number of trusted reverse proxies for `X-Forwarded-*` header processing. |
|
||||
| `ALLOWED_REDIRECT_HOSTS` | `""` | Comma-separated whitelist of safe redirect targets. Empty allows only same-host redirects. |
|
||||
| `ALLOW_INTERNAL_ENDPOINTS` | `false` | Allow connections to internal/private IPs for webhooks and replication targets. **Keep disabled in production unless needed.** |
|
||||
|
||||
## Integrity Scanner
|
||||
|
||||
The integrity scanner detects and optionally auto-repairs data inconsistencies: corrupted objects (ETag mismatch), orphaned files without metadata, phantom metadata without files, stale version archives, ETag cache drift, and unmigrated legacy `.meta.json` files.
|
||||
|
||||
### Enabling Integrity Scanner
|
||||
|
||||
By default, the integrity scanner is disabled. Enable it by setting:
|
||||
|
||||
```bash
|
||||
INTEGRITY_ENABLED=true python run.py
|
||||
```
|
||||
|
||||
Or in your `myfsio.env` file:
|
||||
```
|
||||
INTEGRITY_ENABLED=true
|
||||
INTEGRITY_INTERVAL_HOURS=24 # Run every 24 hours (default)
|
||||
INTEGRITY_BATCH_SIZE=1000 # Max objects to scan per cycle
|
||||
INTEGRITY_AUTO_HEAL=false # Automatically repair detected issues
|
||||
INTEGRITY_DRY_RUN=false # Set to true to log without healing
|
||||
```
|
||||
|
||||
### What Gets Checked
|
||||
|
||||
| Check | Detection | Heal Action |
|
||||
|-------|-----------|-------------|
|
||||
| **Corrupted objects** | File MD5 does not match stored `__etag__` | Update `__etag__` in index (disk data is authoritative) |
|
||||
| **Orphaned objects** | File exists on disk without metadata entry | Create index entry with computed MD5/size/mtime |
|
||||
| **Phantom metadata** | Index entry exists but file is missing from disk | Remove stale entry from `_index.json` |
|
||||
| **Stale versions** | `.json` manifest without `.bin` data or vice versa | Remove orphaned version file |
|
||||
| **ETag cache inconsistency** | `etag_index.json` entry differs from metadata `__etag__` | Delete `etag_index.json` (auto-rebuilt on next list) |
|
||||
| **Legacy metadata drift** | Legacy `.meta.json` differs from index or is unmigrated | Migrate to index and delete legacy file |
|
||||
|
||||
### Admin API
|
||||
|
||||
All integrity endpoints require admin (`iam:*`) permissions.
|
||||
|
||||
| Method | Route | Description |
|
||||
|--------|-------|-------------|
|
||||
| `GET` | `/admin/integrity/status` | Get scanner status and configuration |
|
||||
| `POST` | `/admin/integrity/run` | Trigger a manual scan (body: `{"dry_run": true, "auto_heal": true}`) |
|
||||
| `GET` | `/admin/integrity/history` | Get scan history (query: `?limit=50&offset=0`) |
|
||||
|
||||
### Dry Run Mode
|
||||
|
||||
Set `INTEGRITY_DRY_RUN=true` to log detected issues without making any changes. You can also trigger a one-time dry run via the admin API:
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:5000/admin/integrity/run" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"dry_run": true, "auto_heal": true}'
|
||||
```
|
||||
|
||||
### Configuration Reference
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `INTEGRITY_ENABLED` | `false` | Enable background integrity scanning |
|
||||
| `INTEGRITY_INTERVAL_HOURS` | `24` | Hours between scan cycles |
|
||||
| `INTEGRITY_BATCH_SIZE` | `1000` | Max objects to scan per cycle |
|
||||
| `INTEGRITY_AUTO_HEAL` | `false` | Automatically repair detected issues |
|
||||
| `INTEGRITY_DRY_RUN` | `false` | Log issues without healing |
|
||||
|
||||
## 4. Upgrading and Updates
|
||||
|
||||
### Version Checking
|
||||
@@ -619,11 +753,12 @@ MyFSIO implements a comprehensive Identity and Access Management (IAM) system th
|
||||
|
||||
### Getting Started
|
||||
|
||||
1. On first boot, `data/.myfsio.sys/config/iam.json` is created with a randomly generated admin user. The access key and secret key are printed to the console during first startup. If you miss it, check the `iam.json` file directly—credentials are stored in plaintext.
|
||||
1. On first boot, `data/.myfsio.sys/config/iam.json` is created with a randomly generated admin user. The access key and secret key are printed to the console during first startup. You can set `ADMIN_ACCESS_KEY` and `ADMIN_SECRET_KEY` environment variables to use custom credentials instead of random ones. If `SECRET_KEY` is configured, the IAM config file is encrypted at rest using AES (Fernet). To reset admin credentials later, run `python run.py --reset-cred`.
|
||||
2. Sign into the UI using the generated credentials, then open **IAM**:
|
||||
- **Create user**: supply a display name and optional JSON inline policy array.
|
||||
- **Create user**: supply a display name, optional JSON inline policy array, and optional credential expiry date.
|
||||
- **Set expiry**: assign an expiration date to any user's credentials. Expired credentials are rejected at authentication time. The UI shows expiry badges and preset durations (1h, 24h, 7d, 30d, 90d).
|
||||
- **Rotate secret**: generates a new secret key; the UI surfaces it once.
|
||||
- **Policy editor**: select a user, paste an array of objects (`{"bucket": "*", "actions": ["list", "read"]}`), and submit. Alias support includes AWS-style verbs (e.g., `s3:GetObject`).
|
||||
- **Policy editor**: select a user, paste an array of objects (`{"bucket": "*", "actions": ["list", "read"]}`), and submit. An optional `"prefix"` field restricts object-level actions to a key prefix (e.g., `"uploads/"`). Alias support includes AWS-style verbs (e.g., `s3:GetObject`).
|
||||
3. Wildcard action `iam:*` is supported for admin user definitions.
|
||||
|
||||
> **Breaking Change (v0.2.0+):** Previous versions used fixed default credentials (`localadmin/localadmin`). If upgrading from an older version, your existing credentials remain unchanged, but new installations will generate random credentials.
|
||||
@@ -639,8 +774,11 @@ The API expects every request to include authentication headers. The UI persists
|
||||
|
||||
**Security Features:**
|
||||
- **Lockout Protection**: After `AUTH_MAX_ATTEMPTS` (default: 5) failed login attempts, the account is locked for `AUTH_LOCKOUT_MINUTES` (default: 15 minutes).
|
||||
- **Credential Expiry**: Each user can have an optional `expires_at` timestamp (ISO 8601). Once expired, all API requests using those credentials are rejected. Set or clear expiry via the UI or API.
|
||||
- **IAM Config Encryption**: When `SECRET_KEY` is set, the IAM config file (`iam.json`) is encrypted at rest using Fernet (AES-256-CBC with HMAC). Existing plaintext configs are automatically encrypted on next load.
|
||||
- **Session Management**: UI sessions remain valid for `SESSION_LIFETIME_DAYS` (default: 30 days).
|
||||
- **Hot Reload**: IAM configuration changes take effect immediately without restart.
|
||||
- **Credential Reset**: Run `python run.py --reset-cred` to reset admin credentials. Supports `ADMIN_ACCESS_KEY` and `ADMIN_SECRET_KEY` env vars for deterministic keys.
|
||||
|
||||
### Permission Model
|
||||
|
||||
@@ -659,13 +797,23 @@ Both layers are evaluated for each request. A user must have permission in their
|
||||
| --- | --- | --- |
|
||||
| `list` | List buckets and objects | `s3:ListBucket`, `s3:ListAllMyBuckets`, `s3:ListBucketVersions`, `s3:ListMultipartUploads`, `s3:ListParts` |
|
||||
| `read` | Download objects, get metadata | `s3:GetObject`, `s3:GetObjectVersion`, `s3:GetObjectTagging`, `s3:GetObjectVersionTagging`, `s3:GetObjectAcl`, `s3:GetBucketVersioning`, `s3:HeadObject`, `s3:HeadBucket` |
|
||||
| `write` | Upload objects, create buckets, manage tags | `s3:PutObject`, `s3:CreateBucket`, `s3:PutObjectTagging`, `s3:PutBucketVersioning`, `s3:CreateMultipartUpload`, `s3:UploadPart`, `s3:CompleteMultipartUpload`, `s3:AbortMultipartUpload`, `s3:CopyObject` |
|
||||
| `delete` | Remove objects, versions, and buckets | `s3:DeleteObject`, `s3:DeleteObjectVersion`, `s3:DeleteBucket`, `s3:DeleteObjectTagging` |
|
||||
| `write` | Upload objects, manage object tags | `s3:PutObject`, `s3:PutObjectTagging`, `s3:CreateMultipartUpload`, `s3:UploadPart`, `s3:CompleteMultipartUpload`, `s3:AbortMultipartUpload`, `s3:CopyObject` |
|
||||
| `delete` | Remove objects and versions | `s3:DeleteObject`, `s3:DeleteObjectVersion`, `s3:DeleteObjectTagging` |
|
||||
| `create_bucket` | Create new buckets | `s3:CreateBucket` |
|
||||
| `delete_bucket` | Delete buckets | `s3:DeleteBucket` |
|
||||
| `share` | Manage Access Control Lists (ACLs) | `s3:PutObjectAcl`, `s3:PutBucketAcl`, `s3:GetBucketAcl` |
|
||||
| `policy` | Manage bucket policies | `s3:PutBucketPolicy`, `s3:GetBucketPolicy`, `s3:DeleteBucketPolicy` |
|
||||
| `versioning` | Manage bucket versioning configuration | `s3:GetBucketVersioning`, `s3:PutBucketVersioning` |
|
||||
| `tagging` | Manage bucket-level tags | `s3:GetBucketTagging`, `s3:PutBucketTagging`, `s3:DeleteBucketTagging` |
|
||||
| `encryption` | Manage bucket encryption configuration | `s3:GetEncryptionConfiguration`, `s3:PutEncryptionConfiguration`, `s3:DeleteEncryptionConfiguration` |
|
||||
| `lifecycle` | Manage lifecycle rules | `s3:GetLifecycleConfiguration`, `s3:PutLifecycleConfiguration`, `s3:DeleteLifecycleConfiguration`, `s3:GetBucketLifecycle`, `s3:PutBucketLifecycle` |
|
||||
| `cors` | Manage CORS configuration | `s3:GetBucketCors`, `s3:PutBucketCors`, `s3:DeleteBucketCors` |
|
||||
| `replication` | Configure and manage replication | `s3:GetReplicationConfiguration`, `s3:PutReplicationConfiguration`, `s3:DeleteReplicationConfiguration`, `s3:ReplicateObject`, `s3:ReplicateTags`, `s3:ReplicateDelete` |
|
||||
| `quota` | Manage bucket storage quotas | `s3:GetBucketQuota`, `s3:PutBucketQuota`, `s3:DeleteBucketQuota` |
|
||||
| `object_lock` | Manage object lock, retention, and legal holds | `s3:GetObjectLockConfiguration`, `s3:PutObjectLockConfiguration`, `s3:PutObjectRetention`, `s3:GetObjectRetention`, `s3:PutObjectLegalHold`, `s3:GetObjectLegalHold` |
|
||||
| `notification` | Manage bucket event notifications | `s3:GetBucketNotificationConfiguration`, `s3:PutBucketNotificationConfiguration`, `s3:DeleteBucketNotificationConfiguration` |
|
||||
| `logging` | Manage bucket access logging | `s3:GetBucketLogging`, `s3:PutBucketLogging`, `s3:DeleteBucketLogging` |
|
||||
| `website` | Manage static website hosting configuration | `s3:GetBucketWebsite`, `s3:PutBucketWebsite`, `s3:DeleteBucketWebsite` |
|
||||
|
||||
#### IAM Actions (User Management)
|
||||
|
||||
@@ -676,25 +824,31 @@ Both layers are evaluated for each request. A user must have permission in their
|
||||
| `iam:delete_user` | Delete IAM users | `iam:DeleteUser` |
|
||||
| `iam:rotate_key` | Rotate user secret keys | `iam:RotateAccessKey` |
|
||||
| `iam:update_policy` | Modify user policies | `iam:PutUserPolicy` |
|
||||
| `iam:create_key` | Create additional access keys for a user | `iam:CreateAccessKey` |
|
||||
| `iam:delete_key` | Delete an access key from a user | `iam:DeleteAccessKey` |
|
||||
| `iam:get_user` | View user details and access keys | `iam:GetUser` |
|
||||
| `iam:get_policy` | View user policy configuration | `iam:GetPolicy` |
|
||||
| `iam:disable_user` | Temporarily disable/enable a user account | `iam:DisableUser` |
|
||||
| `iam:*` | **Admin wildcard** – grants all IAM actions | — |
|
||||
|
||||
#### Wildcards
|
||||
|
||||
| Wildcard | Scope | Description |
|
||||
| --- | --- | --- |
|
||||
| `*` (in actions) | All S3 actions | Grants `list`, `read`, `write`, `delete`, `share`, `policy`, `lifecycle`, `cors`, `replication` |
|
||||
| `*` (in actions) | All S3 actions | Grants all 19 S3 actions including `list`, `read`, `write`, `delete`, `create_bucket`, `delete_bucket`, `share`, `policy`, `versioning`, `tagging`, `encryption`, `lifecycle`, `cors`, `replication`, `quota`, `object_lock`, `notification`, `logging`, `website` |
|
||||
| `iam:*` | All IAM actions | Grants all `iam:*` actions for user management |
|
||||
| `*` (in bucket) | All buckets | Policy applies to every bucket |
|
||||
|
||||
### IAM Policy Structure
|
||||
|
||||
User policies are stored as a JSON array of policy objects. Each object specifies a bucket and the allowed actions:
|
||||
User policies are stored as a JSON array of policy objects. Each object specifies a bucket, the allowed actions, and an optional prefix for object-level scoping:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"bucket": "<bucket-name-or-wildcard>",
|
||||
"actions": ["<action1>", "<action2>", ...]
|
||||
"actions": ["<action1>", "<action2>", ...],
|
||||
"prefix": "<optional-key-prefix>"
|
||||
}
|
||||
]
|
||||
```
|
||||
@@ -702,12 +856,13 @@ User policies are stored as a JSON array of policy objects. Each object specifie
|
||||
**Fields:**
|
||||
- `bucket`: The bucket name (case-insensitive) or `*` for all buckets
|
||||
- `actions`: Array of action strings (simple names or AWS aliases)
|
||||
- `prefix`: *(optional)* Restrict object-level actions to keys starting with this prefix. Defaults to `*` (all objects). Example: `"uploads/"` restricts to keys under `uploads/`
|
||||
|
||||
### Example User Policies
|
||||
|
||||
**Full Administrator (complete system access):**
|
||||
```json
|
||||
[{"bucket": "*", "actions": ["list", "read", "write", "delete", "share", "policy", "lifecycle", "cors", "replication", "iam:*"]}]
|
||||
[{"bucket": "*", "actions": ["list", "read", "write", "delete", "share", "policy", "create_bucket", "delete_bucket", "versioning", "tagging", "encryption", "lifecycle", "cors", "replication", "quota", "object_lock", "notification", "logging", "website", "iam:*"]}]
|
||||
```
|
||||
|
||||
**Read-Only User (browse and download only):**
|
||||
@@ -720,6 +875,11 @@ User policies are stored as a JSON array of policy objects. Each object specifie
|
||||
[{"bucket": "user-bucket", "actions": ["list", "read", "write", "delete"]}]
|
||||
```
|
||||
|
||||
**Operator (data operations + bucket management, no config):**
|
||||
```json
|
||||
[{"bucket": "*", "actions": ["list", "read", "write", "delete", "create_bucket", "delete_bucket"]}]
|
||||
```
|
||||
|
||||
**Multiple Bucket Access (different permissions per bucket):**
|
||||
```json
|
||||
[
|
||||
@@ -729,9 +889,14 @@ User policies are stored as a JSON array of policy objects. Each object specifie
|
||||
]
|
||||
```
|
||||
|
||||
**Prefix-Scoped Access (restrict to a folder inside a shared bucket):**
|
||||
```json
|
||||
[{"bucket": "shared-data", "actions": ["list", "read", "write", "delete"], "prefix": "team-a/"}]
|
||||
```
|
||||
|
||||
**IAM Manager (manage users but no data access):**
|
||||
```json
|
||||
[{"bucket": "*", "actions": ["iam:list_users", "iam:create_user", "iam:delete_user", "iam:rotate_key", "iam:update_policy"]}]
|
||||
[{"bucket": "*", "actions": ["iam:list_users", "iam:create_user", "iam:delete_user", "iam:rotate_key", "iam:update_policy", "iam:create_key", "iam:delete_key", "iam:get_user", "iam:get_policy", "iam:disable_user"]}]
|
||||
```
|
||||
|
||||
**Replication Operator (manage replication only):**
|
||||
@@ -751,10 +916,10 @@ User policies are stored as a JSON array of policy objects. Each object specifie
|
||||
|
||||
**Bucket Administrator (full bucket config, no IAM access):**
|
||||
```json
|
||||
[{"bucket": "my-bucket", "actions": ["list", "read", "write", "delete", "policy", "lifecycle", "cors"]}]
|
||||
[{"bucket": "my-bucket", "actions": ["list", "read", "write", "delete", "create_bucket", "delete_bucket", "share", "policy", "versioning", "tagging", "encryption", "lifecycle", "cors", "replication", "quota", "object_lock", "notification", "logging", "website"]}]
|
||||
```
|
||||
|
||||
**Upload-Only User (write but cannot read back):**
|
||||
**Upload-Only User (write but cannot create/delete buckets):**
|
||||
```json
|
||||
[{"bucket": "drop-box", "actions": ["write"]}]
|
||||
```
|
||||
@@ -800,7 +965,8 @@ curl -X POST http://localhost:5000/iam/users \
|
||||
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
|
||||
-d '{
|
||||
"display_name": "New User",
|
||||
"policies": [{"bucket": "*", "actions": ["list", "read"]}]
|
||||
"policies": [{"bucket": "*", "actions": ["list", "read"]}],
|
||||
"expires_at": "2026-12-31T23:59:59Z"
|
||||
}'
|
||||
|
||||
# Rotate user secret (requires iam:rotate_key)
|
||||
@@ -813,9 +979,45 @@ curl -X PUT http://localhost:5000/iam/users/<access-key>/policies \
|
||||
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
|
||||
-d '[{"bucket": "*", "actions": ["list", "read", "write"]}]'
|
||||
|
||||
# Update credential expiry (requires iam:update_policy)
|
||||
curl -X POST http://localhost:5000/iam/users/<access-key>/expiry \
|
||||
-H "Content-Type: application/x-www-form-urlencoded" \
|
||||
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
|
||||
-d 'expires_at=2026-12-31T23:59:59Z'
|
||||
|
||||
# Remove credential expiry (never expires)
|
||||
curl -X POST http://localhost:5000/iam/users/<access-key>/expiry \
|
||||
-H "Content-Type: application/x-www-form-urlencoded" \
|
||||
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
|
||||
-d 'expires_at='
|
||||
|
||||
# Delete a user (requires iam:delete_user)
|
||||
curl -X DELETE http://localhost:5000/iam/users/<access-key> \
|
||||
-H "X-Access-Key: ..." -H "X-Secret-Key: ..."
|
||||
|
||||
# Get user details (requires iam:get_user) — via Admin API
|
||||
curl http://localhost:5000/admin/iam/users/<user-id-or-access-key> \
|
||||
-H "Authorization: AWS4-HMAC-SHA256 ..."
|
||||
|
||||
# Get user policies (requires iam:get_policy) — via Admin API
|
||||
curl http://localhost:5000/admin/iam/users/<user-id-or-access-key>/policies \
|
||||
-H "Authorization: AWS4-HMAC-SHA256 ..."
|
||||
|
||||
# Create additional access key for a user (requires iam:create_key)
|
||||
curl -X POST http://localhost:5000/admin/iam/users/<user-id-or-access-key>/keys \
|
||||
-H "Authorization: AWS4-HMAC-SHA256 ..."
|
||||
|
||||
# Delete an access key (requires iam:delete_key)
|
||||
curl -X DELETE http://localhost:5000/admin/iam/users/<user-id>/keys/<access-key> \
|
||||
-H "Authorization: AWS4-HMAC-SHA256 ..."
|
||||
|
||||
# Disable a user account (requires iam:disable_user)
|
||||
curl -X POST http://localhost:5000/admin/iam/users/<user-id-or-access-key>/disable \
|
||||
-H "Authorization: AWS4-HMAC-SHA256 ..."
|
||||
|
||||
# Re-enable a user account (requires iam:disable_user)
|
||||
curl -X POST http://localhost:5000/admin/iam/users/<user-id-or-access-key>/enable \
|
||||
-H "Authorization: AWS4-HMAC-SHA256 ..."
|
||||
```
|
||||
|
||||
### Permission Precedence
|
||||
@@ -824,8 +1026,9 @@ When a request is made, permissions are evaluated in this order:
|
||||
|
||||
1. **Authentication** – Verify the access key and secret key are valid
|
||||
2. **Lockout Check** – Ensure the account is not locked due to failed attempts
|
||||
3. **IAM Policy Check** – Verify the user has the required action for the target bucket
|
||||
4. **Bucket Policy Check** – If a bucket policy exists, verify it allows the action
|
||||
3. **Expiry Check** – Reject requests if the user's credentials have expired (`expires_at`)
|
||||
4. **IAM Policy Check** – Verify the user has the required action for the target bucket
|
||||
5. **Bucket Policy Check** – If a bucket policy exists, verify it allows the action
|
||||
|
||||
A request is allowed only if:
|
||||
- The IAM policy grants the action, AND
|
||||
@@ -912,7 +1115,7 @@ Objects with forward slashes (`/`) in their keys are displayed as a folder hiera
|
||||
|
||||
- Select multiple objects using checkboxes
|
||||
- **Bulk Delete**: Delete multiple objects at once
|
||||
- **Bulk Download**: Download selected objects as individual files
|
||||
- **Bulk Download**: Download selected objects as a single ZIP archive (up to `BULK_DOWNLOAD_MAX_BYTES`, default 1 GiB)
|
||||
|
||||
#### Search & Filter
|
||||
|
||||
@@ -985,6 +1188,7 @@ MyFSIO supports **server-side encryption at rest** to protect your data. When en
|
||||
|------|-------------|
|
||||
| **AES-256 (SSE-S3)** | Server-managed encryption using a local master key |
|
||||
| **KMS (SSE-KMS)** | Encryption using customer-managed keys via the built-in KMS |
|
||||
| **SSE-C** | Server-side encryption with customer-provided keys (per-request) |
|
||||
|
||||
### Enabling Encryption
|
||||
|
||||
@@ -1083,6 +1287,44 @@ encrypted, metadata = ClientEncryptionHelper.encrypt_for_upload(plaintext, key)
|
||||
decrypted = ClientEncryptionHelper.decrypt_from_download(encrypted, metadata, key)
|
||||
```
|
||||
|
||||
### SSE-C (Customer-Provided Keys)
|
||||
|
||||
With SSE-C, you provide your own 256-bit AES encryption key with each request. The server encrypts/decrypts using your key but never stores it. You must supply the same key for both upload and download.
|
||||
|
||||
**Required headers:**
|
||||
|
||||
| Header | Value |
|
||||
|--------|-------|
|
||||
| `x-amz-server-side-encryption-customer-algorithm` | `AES256` |
|
||||
| `x-amz-server-side-encryption-customer-key` | Base64-encoded 256-bit key |
|
||||
| `x-amz-server-side-encryption-customer-key-MD5` | Base64-encoded MD5 of the key |
|
||||
|
||||
```bash
|
||||
# Generate a 256-bit key
|
||||
KEY=$(openssl rand -base64 32)
|
||||
KEY_MD5=$(echo -n "$KEY" | base64 -d | openssl dgst -md5 -binary | base64)
|
||||
|
||||
# Upload with SSE-C
|
||||
curl -X PUT "http://localhost:5000/my-bucket/secret.txt" \
|
||||
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
|
||||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||||
-H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5" \
|
||||
--data-binary @secret.txt
|
||||
|
||||
# Download with SSE-C (same key required)
|
||||
curl "http://localhost:5000/my-bucket/secret.txt" \
|
||||
-H "X-Access-Key: ..." -H "X-Secret-Key: ..." \
|
||||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||||
-H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5"
|
||||
```
|
||||
|
||||
**Key points:**
|
||||
- SSE-C does not require `ENCRYPTION_ENABLED` or `KMS_ENABLED` — the key is provided per-request
|
||||
- If you lose your key, the data is irrecoverable
|
||||
- The MD5 header is optional but recommended for integrity verification
|
||||
|
||||
### Important Notes
|
||||
|
||||
- **Existing objects are NOT encrypted** - Only new uploads after enabling encryption are encrypted
|
||||
@@ -1552,6 +1794,9 @@ GET /<bucket>?notification # Get event notifications
|
||||
PUT /<bucket>?notification # Set event notifications (webhooks)
|
||||
GET /<bucket>?object-lock # Get object lock configuration
|
||||
PUT /<bucket>?object-lock # Set object lock configuration
|
||||
GET /<bucket>?website # Get website configuration
|
||||
PUT /<bucket>?website # Set website configuration
|
||||
DELETE /<bucket>?website # Delete website configuration
|
||||
GET /<bucket>?uploads # List active multipart uploads
|
||||
GET /<bucket>?versions # List object versions
|
||||
GET /<bucket>?location # Get bucket location/region
|
||||
@@ -1596,6 +1841,11 @@ PUT /admin/sites/<site_id> # Update peer site
|
||||
DELETE /admin/sites/<site_id> # Unregister peer site
|
||||
GET /admin/sites/<site_id>/health # Check peer health
|
||||
GET /admin/topology # Get cluster topology
|
||||
GET /admin/website-domains # List domain mappings
|
||||
POST /admin/website-domains # Create domain mapping
|
||||
GET /admin/website-domains/<domain> # Get domain mapping
|
||||
PUT /admin/website-domains/<domain> # Update domain mapping
|
||||
DELETE /admin/website-domains/<domain> # Delete domain mapping
|
||||
|
||||
# KMS API
|
||||
GET /kms/keys # List KMS keys
|
||||
@@ -1951,6 +2201,20 @@ curl -X PUT "http://localhost:5000/my-bucket/file.txt" \
|
||||
-H "x-amz-meta-newkey: newvalue"
|
||||
```
|
||||
|
||||
### MoveObject (UI)
|
||||
|
||||
Move an object to a different key or bucket. This is a UI-only convenience operation that performs a copy followed by a delete of the source. Requires `read` and `delete` on the source, and `write` on the destination.
|
||||
|
||||
```bash
|
||||
# Move via UI API
|
||||
curl -X POST "http://localhost:5100/ui/buckets/my-bucket/objects/old-path/file.txt/move" \
|
||||
-H "Content-Type: application/json" \
|
||||
--cookie "session=..." \
|
||||
-d '{"dest_bucket": "other-bucket", "dest_key": "new-path/file.txt"}'
|
||||
```
|
||||
|
||||
The move is atomic from the caller's perspective: if the copy succeeds but the delete fails, the object exists in both locations (no data loss).
|
||||
|
||||
### UploadPartCopy
|
||||
|
||||
Copy data from an existing object into a multipart upload part:
|
||||
@@ -2229,3 +2493,113 @@ curl "http://localhost:5000/my-bucket?list-type=2&start-after=photos/2024/" \
|
||||
| `start-after` | Start listing after this key |
|
||||
| `fetch-owner` | Include owner info in response |
|
||||
| `encoding-type` | Set to `url` for URL-encoded keys
|
||||
|
||||
## 26. Static Website Hosting
|
||||
|
||||
MyFSIO can serve S3 buckets as static websites via custom domain mappings. When a request arrives with a `Host` header matching a mapped domain, MyFSIO resolves the bucket and serves objects directly.
|
||||
|
||||
### Enabling
|
||||
|
||||
Set the environment variable:
|
||||
|
||||
```bash
|
||||
WEBSITE_HOSTING_ENABLED=true
|
||||
```
|
||||
|
||||
When disabled, all website hosting endpoints return 400 and domain-based serving is skipped.
|
||||
|
||||
### Configuration
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `WEBSITE_HOSTING_ENABLED` | `false` | Master switch for website hosting |
|
||||
|
||||
### Setting Up a Website
|
||||
|
||||
**Step 1: Configure the bucket website settings**
|
||||
|
||||
```bash
|
||||
curl -X PUT "http://localhost:5000/my-site?website" \
|
||||
-H "Authorization: ..." \
|
||||
-d '<?xml version="1.0" encoding="UTF-8"?>
|
||||
<WebsiteConfiguration>
|
||||
<IndexDocument><Suffix>index.html</Suffix></IndexDocument>
|
||||
<ErrorDocument><Key>404.html</Key></ErrorDocument>
|
||||
</WebsiteConfiguration>'
|
||||
```
|
||||
|
||||
- `IndexDocument` with `Suffix` is required (must not contain `/`)
|
||||
- `ErrorDocument` is optional
|
||||
|
||||
**Step 2: Map a domain to the bucket**
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:5000/admin/website-domains" \
|
||||
-H "Authorization: ..." \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"domain": "example.com", "bucket": "my-site"}'
|
||||
```
|
||||
|
||||
**Step 3: Point your domain to MyFSIO**
|
||||
|
||||
For HTTP-only (direct access), point DNS to the MyFSIO host on port 5000.
|
||||
|
||||
For HTTPS (recommended), use a reverse proxy. The critical requirement is passing the original `Host` header so MyFSIO can match the domain to a bucket.
|
||||
|
||||
**nginx example:**
|
||||
|
||||
```nginx
|
||||
server {
|
||||
server_name example.com;
|
||||
listen 443 ssl;
|
||||
|
||||
ssl_certificate /etc/ssl/certs/example.com.pem;
|
||||
ssl_certificate_key /etc/ssl/private/example.com.key;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:5000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`proxy_set_header Host $host;` is required — without it, MyFSIO cannot match the incoming domain to a bucket. You do not need any path-based routing rules; MyFSIO handles all object resolution internally.
|
||||
|
||||
### How Domain Routing Works
|
||||
|
||||
1. A request arrives with `Host: example.com`
|
||||
2. MyFSIO's `before_request` hook strips the port and looks up the domain in the `WebsiteDomainStore`
|
||||
3. If a match is found, it loads the bucket's website config (index/error documents)
|
||||
4. Object key resolution:
|
||||
- `/` or trailing `/` → append `index_document` (e.g., `index.html`)
|
||||
- `/path` → try exact match, then try `path/index_document`
|
||||
- Not found → serve `error_document` with 404 status
|
||||
5. If no domain match is found, the request falls through to normal S3 API / UI routing
|
||||
|
||||
### Domain Mapping Admin API
|
||||
|
||||
All endpoints require admin (`iam:*`) permissions.
|
||||
|
||||
| Method | Route | Body | Description |
|
||||
|--------|-------|------|-------------|
|
||||
| `GET` | `/admin/website-domains` | — | List all mappings |
|
||||
| `POST` | `/admin/website-domains` | `{"domain": "...", "bucket": "..."}` | Create mapping |
|
||||
| `GET` | `/admin/website-domains/<domain>` | — | Get single mapping |
|
||||
| `PUT` | `/admin/website-domains/<domain>` | `{"bucket": "..."}` | Update mapping |
|
||||
| `DELETE` | `/admin/website-domains/<domain>` | — | Delete mapping |
|
||||
|
||||
### Bucket Website API
|
||||
|
||||
| Method | Route | Description |
|
||||
|--------|-------|-------------|
|
||||
| `PUT` | `/<bucket>?website` | Set website config (XML body) |
|
||||
| `GET` | `/<bucket>?website` | Get website config (XML response) |
|
||||
| `DELETE` | `/<bucket>?website` | Remove website config |
|
||||
|
||||
### Web UI
|
||||
|
||||
- **Per-bucket config:** Bucket Details → Properties tab → "Static Website Hosting" card
|
||||
- **Domain management:** Sidebar → "Domains" (visible when hosting is enabled and user is admin)
|
||||
|
||||
24
myfsio_core/Cargo.toml
Normal file
24
myfsio_core/Cargo.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "myfsio_core"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
name = "myfsio_core"
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
pyo3 = { version = "0.28", features = ["extension-module"] }
|
||||
hmac = "0.12"
|
||||
sha2 = "0.10"
|
||||
md-5 = "0.10"
|
||||
hex = "0.4"
|
||||
unicode-normalization = "0.1"
|
||||
serde_json = "1"
|
||||
regex = "1"
|
||||
lru = "0.14"
|
||||
parking_lot = "0.12"
|
||||
percent-encoding = "2"
|
||||
aes-gcm = "0.10"
|
||||
hkdf = "0.12"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
11
myfsio_core/pyproject.toml
Normal file
11
myfsio_core/pyproject.toml
Normal file
@@ -0,0 +1,11 @@
|
||||
[build-system]
|
||||
requires = ["maturin>=1.0,<2.0"]
|
||||
build-backend = "maturin"
|
||||
|
||||
[project]
|
||||
name = "myfsio_core"
|
||||
version = "0.1.0"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
[tool.maturin]
|
||||
features = ["pyo3/extension-module"]
|
||||
192
myfsio_core/src/crypto.rs
Normal file
192
myfsio_core/src/crypto.rs
Normal file
@@ -0,0 +1,192 @@
|
||||
use aes_gcm::aead::Aead;
|
||||
use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
|
||||
use hkdf::Hkdf;
|
||||
use pyo3::exceptions::{PyIOError, PyValueError};
|
||||
use pyo3::prelude::*;
|
||||
use sha2::Sha256;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
|
||||
const DEFAULT_CHUNK_SIZE: usize = 65536;
|
||||
const HEADER_SIZE: usize = 4;
|
||||
|
||||
fn read_exact_chunk(reader: &mut impl Read, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
let mut filled = 0;
|
||||
while filled < buf.len() {
|
||||
match reader.read(&mut buf[filled..]) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => filled += n,
|
||||
Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Ok(filled)
|
||||
}
|
||||
|
||||
fn derive_chunk_nonce(base_nonce: &[u8], chunk_index: u32) -> Result<[u8; 12], String> {
|
||||
let hkdf = Hkdf::<Sha256>::new(Some(base_nonce), b"chunk_nonce");
|
||||
let mut okm = [0u8; 12];
|
||||
hkdf.expand(&chunk_index.to_be_bytes(), &mut okm)
|
||||
.map_err(|e| format!("HKDF expand failed: {}", e))?;
|
||||
Ok(okm)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (input_path, output_path, key, base_nonce, chunk_size=DEFAULT_CHUNK_SIZE))]
|
||||
pub fn encrypt_stream_chunked(
|
||||
py: Python<'_>,
|
||||
input_path: &str,
|
||||
output_path: &str,
|
||||
key: &[u8],
|
||||
base_nonce: &[u8],
|
||||
chunk_size: usize,
|
||||
) -> PyResult<u32> {
|
||||
if key.len() != 32 {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"Key must be 32 bytes, got {}",
|
||||
key.len()
|
||||
)));
|
||||
}
|
||||
if base_nonce.len() != 12 {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"Base nonce must be 12 bytes, got {}",
|
||||
base_nonce.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let chunk_size = if chunk_size == 0 {
|
||||
DEFAULT_CHUNK_SIZE
|
||||
} else {
|
||||
chunk_size
|
||||
};
|
||||
|
||||
let inp = input_path.to_owned();
|
||||
let out = output_path.to_owned();
|
||||
let key_arr: [u8; 32] = key.try_into().unwrap();
|
||||
let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap();
|
||||
|
||||
py.detach(move || {
|
||||
let cipher = Aes256Gcm::new(&key_arr.into());
|
||||
|
||||
let mut infile = File::open(&inp)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to open input: {}", e)))?;
|
||||
let mut outfile = File::create(&out)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to create output: {}", e)))?;
|
||||
|
||||
outfile
|
||||
.write_all(&[0u8; 4])
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to write header: {}", e)))?;
|
||||
|
||||
let mut buf = vec![0u8; chunk_size];
|
||||
let mut chunk_index: u32 = 0;
|
||||
|
||||
loop {
|
||||
let n = read_exact_chunk(&mut infile, &mut buf)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to read: {}", e)))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)
|
||||
.map_err(|e| PyValueError::new_err(e))?;
|
||||
let nonce = Nonce::from_slice(&nonce_bytes);
|
||||
|
||||
let encrypted = cipher
|
||||
.encrypt(nonce, &buf[..n])
|
||||
.map_err(|e| PyValueError::new_err(format!("Encrypt failed: {}", e)))?;
|
||||
|
||||
let size = encrypted.len() as u32;
|
||||
outfile
|
||||
.write_all(&size.to_be_bytes())
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to write chunk size: {}", e)))?;
|
||||
outfile
|
||||
.write_all(&encrypted)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to write chunk: {}", e)))?;
|
||||
|
||||
chunk_index += 1;
|
||||
}
|
||||
|
||||
outfile
|
||||
.seek(SeekFrom::Start(0))
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to seek: {}", e)))?;
|
||||
outfile
|
||||
.write_all(&chunk_index.to_be_bytes())
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to write chunk count: {}", e)))?;
|
||||
|
||||
Ok(chunk_index)
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn decrypt_stream_chunked(
|
||||
py: Python<'_>,
|
||||
input_path: &str,
|
||||
output_path: &str,
|
||||
key: &[u8],
|
||||
base_nonce: &[u8],
|
||||
) -> PyResult<u32> {
|
||||
if key.len() != 32 {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"Key must be 32 bytes, got {}",
|
||||
key.len()
|
||||
)));
|
||||
}
|
||||
if base_nonce.len() != 12 {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"Base nonce must be 12 bytes, got {}",
|
||||
base_nonce.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let inp = input_path.to_owned();
|
||||
let out = output_path.to_owned();
|
||||
let key_arr: [u8; 32] = key.try_into().unwrap();
|
||||
let nonce_arr: [u8; 12] = base_nonce.try_into().unwrap();
|
||||
|
||||
py.detach(move || {
|
||||
let cipher = Aes256Gcm::new(&key_arr.into());
|
||||
|
||||
let mut infile = File::open(&inp)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to open input: {}", e)))?;
|
||||
let mut outfile = File::create(&out)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to create output: {}", e)))?;
|
||||
|
||||
let mut header = [0u8; HEADER_SIZE];
|
||||
infile
|
||||
.read_exact(&mut header)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to read header: {}", e)))?;
|
||||
let chunk_count = u32::from_be_bytes(header);
|
||||
|
||||
let mut size_buf = [0u8; HEADER_SIZE];
|
||||
for chunk_index in 0..chunk_count {
|
||||
infile
|
||||
.read_exact(&mut size_buf)
|
||||
.map_err(|e| {
|
||||
PyIOError::new_err(format!(
|
||||
"Failed to read chunk {} size: {}",
|
||||
chunk_index, e
|
||||
))
|
||||
})?;
|
||||
let chunk_size = u32::from_be_bytes(size_buf) as usize;
|
||||
|
||||
let mut encrypted = vec![0u8; chunk_size];
|
||||
infile.read_exact(&mut encrypted).map_err(|e| {
|
||||
PyIOError::new_err(format!("Failed to read chunk {}: {}", chunk_index, e))
|
||||
})?;
|
||||
|
||||
let nonce_bytes = derive_chunk_nonce(&nonce_arr, chunk_index)
|
||||
.map_err(|e| PyValueError::new_err(e))?;
|
||||
let nonce = Nonce::from_slice(&nonce_bytes);
|
||||
|
||||
let decrypted = cipher.decrypt(nonce, encrypted.as_ref()).map_err(|e| {
|
||||
PyValueError::new_err(format!("Decrypt chunk {} failed: {}", chunk_index, e))
|
||||
})?;
|
||||
|
||||
outfile.write_all(&decrypted).map_err(|e| {
|
||||
PyIOError::new_err(format!("Failed to write chunk {}: {}", chunk_index, e))
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(chunk_count)
|
||||
})
|
||||
}
|
||||
90
myfsio_core/src/hashing.rs
Normal file
90
myfsio_core/src/hashing.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
use md5::{Digest, Md5};
|
||||
use pyo3::exceptions::PyIOError;
|
||||
use pyo3::prelude::*;
|
||||
use sha2::Sha256;
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
|
||||
const CHUNK_SIZE: usize = 65536;
|
||||
|
||||
#[pyfunction]
|
||||
pub fn md5_file(py: Python<'_>, path: &str) -> PyResult<String> {
|
||||
let path = path.to_owned();
|
||||
py.detach(move || {
|
||||
let mut file = File::open(&path)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
|
||||
let mut hasher = Md5::new();
|
||||
let mut buf = vec![0u8; CHUNK_SIZE];
|
||||
loop {
|
||||
let n = file
|
||||
.read(&mut buf)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok(format!("{:x}", hasher.finalize()))
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn md5_bytes(data: &[u8]) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(data);
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn sha256_file(py: Python<'_>, path: &str) -> PyResult<String> {
|
||||
let path = path.to_owned();
|
||||
py.detach(move || {
|
||||
let mut file = File::open(&path)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buf = vec![0u8; CHUNK_SIZE];
|
||||
loop {
|
||||
let n = file
|
||||
.read(&mut buf)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok(format!("{:x}", hasher.finalize()))
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn sha256_bytes(data: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(data);
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn md5_sha256_file(py: Python<'_>, path: &str) -> PyResult<(String, String)> {
|
||||
let path = path.to_owned();
|
||||
py.detach(move || {
|
||||
let mut file = File::open(&path)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
|
||||
let mut md5_hasher = Md5::new();
|
||||
let mut sha_hasher = Sha256::new();
|
||||
let mut buf = vec![0u8; CHUNK_SIZE];
|
||||
loop {
|
||||
let n = file
|
||||
.read(&mut buf)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
md5_hasher.update(&buf[..n]);
|
||||
sha_hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok((
|
||||
format!("{:x}", md5_hasher.finalize()),
|
||||
format!("{:x}", sha_hasher.finalize()),
|
||||
))
|
||||
})
|
||||
}
|
||||
51
myfsio_core/src/lib.rs
Normal file
51
myfsio_core/src/lib.rs
Normal file
@@ -0,0 +1,51 @@
|
||||
mod crypto;
|
||||
mod hashing;
|
||||
mod metadata;
|
||||
mod sigv4;
|
||||
mod storage;
|
||||
mod streaming;
|
||||
mod validation;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
|
||||
#[pymodule]
|
||||
mod myfsio_core {
|
||||
use super::*;
|
||||
|
||||
#[pymodule_init]
|
||||
fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(sigv4::verify_sigv4_signature, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::derive_signing_key, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::compute_signature, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::build_string_to_sign, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::constant_time_compare, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::clear_signing_key_cache, m)?)?;
|
||||
|
||||
m.add_function(wrap_pyfunction!(hashing::md5_file, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(hashing::md5_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(hashing::sha256_file, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(hashing::sha256_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(hashing::md5_sha256_file, m)?)?;
|
||||
|
||||
m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?;
|
||||
|
||||
m.add_function(wrap_pyfunction!(metadata::read_index_entry, m)?)?;
|
||||
|
||||
m.add_function(wrap_pyfunction!(storage::write_index_entry, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(storage::delete_index_entry, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(storage::check_bucket_contents, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(storage::shallow_scan, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(storage::bucket_stats_scan, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(storage::search_objects_scan, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(storage::build_object_cache, m)?)?;
|
||||
|
||||
m.add_function(wrap_pyfunction!(streaming::stream_to_file_with_md5, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(streaming::assemble_parts_with_md5, m)?)?;
|
||||
|
||||
m.add_function(wrap_pyfunction!(crypto::encrypt_stream_chunked, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(crypto::decrypt_stream_chunked, m)?)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
71
myfsio_core/src/metadata.rs
Normal file
71
myfsio_core/src/metadata.rs
Normal file
@@ -0,0 +1,71 @@
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyList, PyString};
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
|
||||
const MAX_DEPTH: u32 = 64;
|
||||
|
||||
fn value_to_py(py: Python<'_>, v: &Value, depth: u32) -> PyResult<Py<PyAny>> {
|
||||
if depth > MAX_DEPTH {
|
||||
return Err(PyValueError::new_err("JSON nesting too deep"));
|
||||
}
|
||||
match v {
|
||||
Value::Null => Ok(py.None()),
|
||||
Value::Bool(b) => Ok((*b).into_pyobject(py)?.to_owned().into_any().unbind()),
|
||||
Value::Number(n) => {
|
||||
if let Some(i) = n.as_i64() {
|
||||
Ok(i.into_pyobject(py)?.into_any().unbind())
|
||||
} else if let Some(f) = n.as_f64() {
|
||||
Ok(f.into_pyobject(py)?.into_any().unbind())
|
||||
} else {
|
||||
Ok(py.None())
|
||||
}
|
||||
}
|
||||
Value::String(s) => Ok(PyString::new(py, s).into_any().unbind()),
|
||||
Value::Array(arr) => {
|
||||
let list = PyList::empty(py);
|
||||
for item in arr {
|
||||
list.append(value_to_py(py, item, depth + 1)?)?;
|
||||
}
|
||||
Ok(list.into_any().unbind())
|
||||
}
|
||||
Value::Object(map) => {
|
||||
let dict = PyDict::new(py);
|
||||
for (k, val) in map {
|
||||
dict.set_item(k, value_to_py(py, val, depth + 1)?)?;
|
||||
}
|
||||
Ok(dict.into_any().unbind())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn read_index_entry(
|
||||
py: Python<'_>,
|
||||
path: &str,
|
||||
entry_name: &str,
|
||||
) -> PyResult<Option<Py<PyAny>>> {
|
||||
let path_owned = path.to_owned();
|
||||
let entry_owned = entry_name.to_owned();
|
||||
|
||||
let entry: Option<Value> = py.detach(move || -> PyResult<Option<Value>> {
|
||||
let content = match fs::read_to_string(&path_owned) {
|
||||
Ok(c) => c,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
let parsed: Value = match serde_json::from_str(&content) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
match parsed {
|
||||
Value::Object(mut map) => Ok(map.remove(&entry_owned)),
|
||||
_ => Ok(None),
|
||||
}
|
||||
})?;
|
||||
|
||||
match entry {
|
||||
Some(val) => Ok(Some(value_to_py(py, &val, 0)?)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
193
myfsio_core/src/sigv4.rs
Normal file
193
myfsio_core/src/sigv4.rs
Normal file
@@ -0,0 +1,193 @@
|
||||
use hmac::{Hmac, Mac};
|
||||
use lru::LruCache;
|
||||
use parking_lot::Mutex;
|
||||
use percent_encoding::{percent_encode, AsciiSet, NON_ALPHANUMERIC};
|
||||
use pyo3::prelude::*;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::LazyLock;
|
||||
use std::time::Instant;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
struct CacheEntry {
|
||||
key: Vec<u8>,
|
||||
created: Instant,
|
||||
}
|
||||
|
||||
static SIGNING_KEY_CACHE: LazyLock<Mutex<LruCache<(String, String, String, String), CacheEntry>>> =
|
||||
LazyLock::new(|| Mutex::new(LruCache::new(NonZeroUsize::new(256).unwrap())));
|
||||
|
||||
const CACHE_TTL_SECS: u64 = 60;
|
||||
|
||||
const AWS_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
|
||||
.remove(b'-')
|
||||
.remove(b'_')
|
||||
.remove(b'.')
|
||||
.remove(b'~');
|
||||
|
||||
fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec<u8> {
|
||||
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid");
|
||||
mac.update(msg);
|
||||
mac.finalize().into_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(data);
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
fn aws_uri_encode(input: &str) -> String {
|
||||
percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string()
|
||||
}
|
||||
|
||||
fn derive_signing_key_cached(
|
||||
secret_key: &str,
|
||||
date_stamp: &str,
|
||||
region: &str,
|
||||
service: &str,
|
||||
) -> Vec<u8> {
|
||||
let cache_key = (
|
||||
secret_key.to_owned(),
|
||||
date_stamp.to_owned(),
|
||||
region.to_owned(),
|
||||
service.to_owned(),
|
||||
);
|
||||
|
||||
{
|
||||
let mut cache = SIGNING_KEY_CACHE.lock();
|
||||
if let Some(entry) = cache.get(&cache_key) {
|
||||
if entry.created.elapsed().as_secs() < CACHE_TTL_SECS {
|
||||
return entry.key.clone();
|
||||
}
|
||||
cache.pop(&cache_key);
|
||||
}
|
||||
}
|
||||
|
||||
let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date_stamp.as_bytes());
|
||||
let k_region = hmac_sha256(&k_date, region.as_bytes());
|
||||
let k_service = hmac_sha256(&k_region, service.as_bytes());
|
||||
let k_signing = hmac_sha256(&k_service, b"aws4_request");
|
||||
|
||||
{
|
||||
let mut cache = SIGNING_KEY_CACHE.lock();
|
||||
cache.put(
|
||||
cache_key,
|
||||
CacheEntry {
|
||||
key: k_signing.clone(),
|
||||
created: Instant::now(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
k_signing
|
||||
}
|
||||
|
||||
fn constant_time_compare_inner(a: &[u8], b: &[u8]) -> bool {
|
||||
if a.len() != b.len() {
|
||||
return false;
|
||||
}
|
||||
let mut result: u8 = 0;
|
||||
for (x, y) in a.iter().zip(b.iter()) {
|
||||
result |= x ^ y;
|
||||
}
|
||||
result == 0
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn verify_sigv4_signature(
|
||||
method: &str,
|
||||
canonical_uri: &str,
|
||||
query_params: Vec<(String, String)>,
|
||||
signed_headers_str: &str,
|
||||
header_values: Vec<(String, String)>,
|
||||
payload_hash: &str,
|
||||
amz_date: &str,
|
||||
date_stamp: &str,
|
||||
region: &str,
|
||||
service: &str,
|
||||
secret_key: &str,
|
||||
provided_signature: &str,
|
||||
) -> bool {
|
||||
let mut sorted_params = query_params;
|
||||
sorted_params.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
|
||||
|
||||
let canonical_query_string = sorted_params
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{}={}", aws_uri_encode(k), aws_uri_encode(v)))
|
||||
.collect::<Vec<_>>()
|
||||
.join("&");
|
||||
|
||||
let mut canonical_headers = String::new();
|
||||
for (name, value) in &header_values {
|
||||
let lower_name = name.to_lowercase();
|
||||
let normalized = value.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
let final_value = if lower_name == "expect" && normalized.is_empty() {
|
||||
"100-continue"
|
||||
} else {
|
||||
&normalized
|
||||
};
|
||||
canonical_headers.push_str(&lower_name);
|
||||
canonical_headers.push(':');
|
||||
canonical_headers.push_str(final_value);
|
||||
canonical_headers.push('\n');
|
||||
}
|
||||
|
||||
let canonical_request = format!(
|
||||
"{}\n{}\n{}\n{}\n{}\n{}",
|
||||
method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str, payload_hash
|
||||
);
|
||||
|
||||
let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, region, service);
|
||||
let cr_hash = sha256_hex(canonical_request.as_bytes());
|
||||
let string_to_sign = format!(
|
||||
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
|
||||
amz_date, credential_scope, cr_hash
|
||||
);
|
||||
|
||||
let signing_key = derive_signing_key_cached(secret_key, date_stamp, region, service);
|
||||
let calculated = hmac_sha256(&signing_key, string_to_sign.as_bytes());
|
||||
let calculated_hex = hex::encode(&calculated);
|
||||
|
||||
constant_time_compare_inner(calculated_hex.as_bytes(), provided_signature.as_bytes())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn derive_signing_key(
|
||||
secret_key: &str,
|
||||
date_stamp: &str,
|
||||
region: &str,
|
||||
service: &str,
|
||||
) -> Vec<u8> {
|
||||
derive_signing_key_cached(secret_key, date_stamp, region, service)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String {
|
||||
let sig = hmac_sha256(signing_key, string_to_sign.as_bytes());
|
||||
hex::encode(sig)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn build_string_to_sign(
|
||||
amz_date: &str,
|
||||
credential_scope: &str,
|
||||
canonical_request: &str,
|
||||
) -> String {
|
||||
let cr_hash = sha256_hex(canonical_request.as_bytes());
|
||||
format!(
|
||||
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
|
||||
amz_date, credential_scope, cr_hash
|
||||
)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn constant_time_compare(a: &str, b: &str) -> bool {
|
||||
constant_time_compare_inner(a.as_bytes(), b.as_bytes())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn clear_signing_key_cache() {
|
||||
SIGNING_KEY_CACHE.lock().clear();
|
||||
}
|
||||
817
myfsio_core/src/storage.rs
Normal file
817
myfsio_core/src/storage.rs
Normal file
@@ -0,0 +1,817 @@
|
||||
use pyo3::exceptions::PyIOError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyList, PyString, PyTuple};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::SystemTime;
|
||||
|
||||
const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
|
||||
|
||||
fn system_time_to_epoch(t: SystemTime) -> f64 {
|
||||
t.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
fn extract_etag_from_meta_bytes(content: &[u8]) -> Option<String> {
|
||||
let marker = b"\"__etag__\"";
|
||||
let idx = content.windows(marker.len()).position(|w| w == marker)?;
|
||||
let after = &content[idx + marker.len()..];
|
||||
let start = after.iter().position(|&b| b == b'"')? + 1;
|
||||
let rest = &after[start..];
|
||||
let end = rest.iter().position(|&b| b == b'"')?;
|
||||
std::str::from_utf8(&rest[..end]).ok().map(|s| s.to_owned())
|
||||
}
|
||||
|
||||
fn has_any_file(root: &str) -> bool {
|
||||
let root_path = Path::new(root);
|
||||
if !root_path.is_dir() {
|
||||
return false;
|
||||
}
|
||||
let mut stack = vec![root_path.to_path_buf()];
|
||||
while let Some(current) = stack.pop() {
|
||||
let entries = match fs::read_dir(¤t) {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for entry_result in entries {
|
||||
let entry = match entry_result {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let ft = match entry.file_type() {
|
||||
Ok(ft) => ft,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if ft.is_file() {
|
||||
return true;
|
||||
}
|
||||
if ft.is_dir() && !ft.is_symlink() {
|
||||
stack.push(entry.path());
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn write_index_entry(
|
||||
py: Python<'_>,
|
||||
path: &str,
|
||||
entry_name: &str,
|
||||
entry_data_json: &str,
|
||||
) -> PyResult<()> {
|
||||
let path_owned = path.to_owned();
|
||||
let entry_owned = entry_name.to_owned();
|
||||
let data_owned = entry_data_json.to_owned();
|
||||
|
||||
py.detach(move || -> PyResult<()> {
|
||||
let entry_value: Value = serde_json::from_str(&data_owned)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to parse entry data: {}", e)))?;
|
||||
|
||||
if let Some(parent) = Path::new(&path_owned).parent() {
|
||||
let _ = fs::create_dir_all(parent);
|
||||
}
|
||||
|
||||
let mut index_data: serde_json::Map<String, Value> = match fs::read_to_string(&path_owned)
|
||||
{
|
||||
Ok(content) => serde_json::from_str(&content).unwrap_or_default(),
|
||||
Err(_) => serde_json::Map::new(),
|
||||
};
|
||||
|
||||
index_data.insert(entry_owned, entry_value);
|
||||
|
||||
let serialized = serde_json::to_string(&Value::Object(index_data))
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to serialize index: {}", e)))?;
|
||||
|
||||
fs::write(&path_owned, serialized)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to write index: {}", e)))?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn delete_index_entry(py: Python<'_>, path: &str, entry_name: &str) -> PyResult<bool> {
|
||||
let path_owned = path.to_owned();
|
||||
let entry_owned = entry_name.to_owned();
|
||||
|
||||
py.detach(move || -> PyResult<bool> {
|
||||
let content = match fs::read_to_string(&path_owned) {
|
||||
Ok(c) => c,
|
||||
Err(_) => return Ok(false),
|
||||
};
|
||||
|
||||
let mut index_data: serde_json::Map<String, Value> =
|
||||
match serde_json::from_str(&content) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Ok(false),
|
||||
};
|
||||
|
||||
if index_data.remove(&entry_owned).is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if index_data.is_empty() {
|
||||
let _ = fs::remove_file(&path_owned);
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let serialized = serde_json::to_string(&Value::Object(index_data))
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to serialize index: {}", e)))?;
|
||||
|
||||
fs::write(&path_owned, serialized)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to write index: {}", e)))?;
|
||||
|
||||
Ok(false)
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn check_bucket_contents(
|
||||
py: Python<'_>,
|
||||
bucket_path: &str,
|
||||
version_roots: Vec<String>,
|
||||
multipart_roots: Vec<String>,
|
||||
) -> PyResult<(bool, bool, bool)> {
|
||||
let bucket_owned = bucket_path.to_owned();
|
||||
|
||||
py.detach(move || -> PyResult<(bool, bool, bool)> {
|
||||
let mut has_objects = false;
|
||||
let bucket_p = Path::new(&bucket_owned);
|
||||
if bucket_p.is_dir() {
|
||||
let mut stack = vec![bucket_p.to_path_buf()];
|
||||
'obj_scan: while let Some(current) = stack.pop() {
|
||||
let is_root = current == bucket_p;
|
||||
let entries = match fs::read_dir(¤t) {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for entry_result in entries {
|
||||
let entry = match entry_result {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let ft = match entry.file_type() {
|
||||
Ok(ft) => ft,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if is_root {
|
||||
if let Some(name) = entry.file_name().to_str() {
|
||||
if INTERNAL_FOLDERS.contains(&name) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ft.is_file() && !ft.is_symlink() {
|
||||
has_objects = true;
|
||||
break 'obj_scan;
|
||||
}
|
||||
if ft.is_dir() && !ft.is_symlink() {
|
||||
stack.push(entry.path());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut has_versions = false;
|
||||
for root in &version_roots {
|
||||
if has_versions {
|
||||
break;
|
||||
}
|
||||
has_versions = has_any_file(root);
|
||||
}
|
||||
|
||||
let mut has_multipart = false;
|
||||
for root in &multipart_roots {
|
||||
if has_multipart {
|
||||
break;
|
||||
}
|
||||
has_multipart = has_any_file(root);
|
||||
}
|
||||
|
||||
Ok((has_objects, has_versions, has_multipart))
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn shallow_scan(
|
||||
py: Python<'_>,
|
||||
target_dir: &str,
|
||||
prefix: &str,
|
||||
meta_cache_json: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let target_owned = target_dir.to_owned();
|
||||
let prefix_owned = prefix.to_owned();
|
||||
let cache_owned = meta_cache_json.to_owned();
|
||||
|
||||
let result: (
|
||||
Vec<(String, u64, f64, Option<String>)>,
|
||||
Vec<String>,
|
||||
Vec<(String, bool)>,
|
||||
) = py.detach(move || -> PyResult<(
|
||||
Vec<(String, u64, f64, Option<String>)>,
|
||||
Vec<String>,
|
||||
Vec<(String, bool)>,
|
||||
)> {
|
||||
let meta_cache: HashMap<String, String> =
|
||||
serde_json::from_str(&cache_owned).unwrap_or_default();
|
||||
|
||||
let mut files: Vec<(String, u64, f64, Option<String>)> = Vec::new();
|
||||
let mut dirs: Vec<String> = Vec::new();
|
||||
|
||||
let entries = match fs::read_dir(&target_owned) {
|
||||
Ok(e) => e,
|
||||
Err(_) => return Ok((files, dirs, Vec::new())),
|
||||
};
|
||||
|
||||
for entry_result in entries {
|
||||
let entry = match entry_result {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let name = match entry.file_name().into_string() {
|
||||
Ok(n) => n,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if INTERNAL_FOLDERS.contains(&name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let ft = match entry.file_type() {
|
||||
Ok(ft) => ft,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if ft.is_dir() && !ft.is_symlink() {
|
||||
let cp = format!("{}{}/", prefix_owned, name);
|
||||
dirs.push(cp);
|
||||
} else if ft.is_file() && !ft.is_symlink() {
|
||||
let key = format!("{}{}", prefix_owned, name);
|
||||
let md = match entry.metadata() {
|
||||
Ok(m) => m,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let size = md.len();
|
||||
let mtime = md
|
||||
.modified()
|
||||
.map(system_time_to_epoch)
|
||||
.unwrap_or(0.0);
|
||||
let etag = meta_cache.get(&key).cloned();
|
||||
files.push((key, size, mtime, etag));
|
||||
}
|
||||
}
|
||||
|
||||
files.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
dirs.sort();
|
||||
|
||||
let mut merged: Vec<(String, bool)> = Vec::with_capacity(files.len() + dirs.len());
|
||||
let mut fi = 0;
|
||||
let mut di = 0;
|
||||
while fi < files.len() && di < dirs.len() {
|
||||
if files[fi].0 <= dirs[di] {
|
||||
merged.push((files[fi].0.clone(), false));
|
||||
fi += 1;
|
||||
} else {
|
||||
merged.push((dirs[di].clone(), true));
|
||||
di += 1;
|
||||
}
|
||||
}
|
||||
while fi < files.len() {
|
||||
merged.push((files[fi].0.clone(), false));
|
||||
fi += 1;
|
||||
}
|
||||
while di < dirs.len() {
|
||||
merged.push((dirs[di].clone(), true));
|
||||
di += 1;
|
||||
}
|
||||
|
||||
Ok((files, dirs, merged))
|
||||
})?;
|
||||
|
||||
let (files, dirs, merged) = result;
|
||||
|
||||
let dict = PyDict::new(py);
|
||||
|
||||
let files_list = PyList::empty(py);
|
||||
for (key, size, mtime, etag) in &files {
|
||||
let etag_py: Py<PyAny> = match etag {
|
||||
Some(e) => PyString::new(py, e).into_any().unbind(),
|
||||
None => py.None(),
|
||||
};
|
||||
let tuple = PyTuple::new(py, &[
|
||||
PyString::new(py, key).into_any().unbind(),
|
||||
size.into_pyobject(py)?.into_any().unbind(),
|
||||
mtime.into_pyobject(py)?.into_any().unbind(),
|
||||
etag_py,
|
||||
])?;
|
||||
files_list.append(tuple)?;
|
||||
}
|
||||
dict.set_item("files", files_list)?;
|
||||
|
||||
let dirs_list = PyList::empty(py);
|
||||
for d in &dirs {
|
||||
dirs_list.append(PyString::new(py, d))?;
|
||||
}
|
||||
dict.set_item("dirs", dirs_list)?;
|
||||
|
||||
let merged_list = PyList::empty(py);
|
||||
for (key, is_dir) in &merged {
|
||||
let bool_obj: Py<PyAny> = if *is_dir {
|
||||
true.into_pyobject(py)?.to_owned().into_any().unbind()
|
||||
} else {
|
||||
false.into_pyobject(py)?.to_owned().into_any().unbind()
|
||||
};
|
||||
let tuple = PyTuple::new(py, &[
|
||||
PyString::new(py, key).into_any().unbind(),
|
||||
bool_obj,
|
||||
])?;
|
||||
merged_list.append(tuple)?;
|
||||
}
|
||||
dict.set_item("merged_keys", merged_list)?;
|
||||
|
||||
Ok(dict.into_any().unbind())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn bucket_stats_scan(
|
||||
py: Python<'_>,
|
||||
bucket_path: &str,
|
||||
versions_root: &str,
|
||||
) -> PyResult<(u64, u64, u64, u64)> {
|
||||
let bucket_owned = bucket_path.to_owned();
|
||||
let versions_owned = versions_root.to_owned();
|
||||
|
||||
py.detach(move || -> PyResult<(u64, u64, u64, u64)> {
|
||||
let mut object_count: u64 = 0;
|
||||
let mut total_bytes: u64 = 0;
|
||||
|
||||
let bucket_p = Path::new(&bucket_owned);
|
||||
if bucket_p.is_dir() {
|
||||
let mut stack = vec![bucket_p.to_path_buf()];
|
||||
while let Some(current) = stack.pop() {
|
||||
let is_root = current == bucket_p;
|
||||
let entries = match fs::read_dir(¤t) {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for entry_result in entries {
|
||||
let entry = match entry_result {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if is_root {
|
||||
if let Some(name) = entry.file_name().to_str() {
|
||||
if INTERNAL_FOLDERS.contains(&name) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
let ft = match entry.file_type() {
|
||||
Ok(ft) => ft,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if ft.is_dir() && !ft.is_symlink() {
|
||||
stack.push(entry.path());
|
||||
} else if ft.is_file() && !ft.is_symlink() {
|
||||
object_count += 1;
|
||||
if let Ok(md) = entry.metadata() {
|
||||
total_bytes += md.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut version_count: u64 = 0;
|
||||
let mut version_bytes: u64 = 0;
|
||||
|
||||
let versions_p = Path::new(&versions_owned);
|
||||
if versions_p.is_dir() {
|
||||
let mut stack = vec![versions_p.to_path_buf()];
|
||||
while let Some(current) = stack.pop() {
|
||||
let entries = match fs::read_dir(¤t) {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for entry_result in entries {
|
||||
let entry = match entry_result {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let ft = match entry.file_type() {
|
||||
Ok(ft) => ft,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if ft.is_dir() && !ft.is_symlink() {
|
||||
stack.push(entry.path());
|
||||
} else if ft.is_file() && !ft.is_symlink() {
|
||||
if let Some(name) = entry.file_name().to_str() {
|
||||
if name.ends_with(".bin") {
|
||||
version_count += 1;
|
||||
if let Ok(md) = entry.metadata() {
|
||||
version_bytes += md.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((object_count, total_bytes, version_count, version_bytes))
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (bucket_path, search_root, query, limit))]
|
||||
pub fn search_objects_scan(
|
||||
py: Python<'_>,
|
||||
bucket_path: &str,
|
||||
search_root: &str,
|
||||
query: &str,
|
||||
limit: usize,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let bucket_owned = bucket_path.to_owned();
|
||||
let search_owned = search_root.to_owned();
|
||||
let query_owned = query.to_owned();
|
||||
|
||||
let result: (Vec<(String, u64, f64)>, bool) = py.detach(
|
||||
move || -> PyResult<(Vec<(String, u64, f64)>, bool)> {
|
||||
let query_lower = query_owned.to_lowercase();
|
||||
let bucket_len = bucket_owned.len() + 1;
|
||||
let scan_limit = limit * 4;
|
||||
let mut matched: usize = 0;
|
||||
let mut results: Vec<(String, u64, f64)> = Vec::new();
|
||||
|
||||
let search_p = Path::new(&search_owned);
|
||||
if !search_p.is_dir() {
|
||||
return Ok((results, false));
|
||||
}
|
||||
|
||||
let bucket_p = Path::new(&bucket_owned);
|
||||
let mut stack = vec![search_p.to_path_buf()];
|
||||
|
||||
'scan: while let Some(current) = stack.pop() {
|
||||
let is_bucket_root = current == bucket_p;
|
||||
let entries = match fs::read_dir(¤t) {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for entry_result in entries {
|
||||
let entry = match entry_result {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if is_bucket_root {
|
||||
if let Some(name) = entry.file_name().to_str() {
|
||||
if INTERNAL_FOLDERS.contains(&name) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
let ft = match entry.file_type() {
|
||||
Ok(ft) => ft,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if ft.is_dir() && !ft.is_symlink() {
|
||||
stack.push(entry.path());
|
||||
} else if ft.is_file() && !ft.is_symlink() {
|
||||
let full_path = entry.path();
|
||||
let full_str = full_path.to_string_lossy();
|
||||
if full_str.len() <= bucket_len {
|
||||
continue;
|
||||
}
|
||||
let key = full_str[bucket_len..].replace('\\', "/");
|
||||
if key.to_lowercase().contains(&query_lower) {
|
||||
if let Ok(md) = entry.metadata() {
|
||||
let size = md.len();
|
||||
let mtime = md
|
||||
.modified()
|
||||
.map(system_time_to_epoch)
|
||||
.unwrap_or(0.0);
|
||||
results.push((key, size, mtime));
|
||||
matched += 1;
|
||||
}
|
||||
}
|
||||
if matched >= scan_limit {
|
||||
break 'scan;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
let truncated = results.len() > limit;
|
||||
results.truncate(limit);
|
||||
|
||||
Ok((results, truncated))
|
||||
},
|
||||
)?;
|
||||
|
||||
let (results, truncated) = result;
|
||||
|
||||
let dict = PyDict::new(py);
|
||||
|
||||
let results_list = PyList::empty(py);
|
||||
for (key, size, mtime) in &results {
|
||||
let tuple = PyTuple::new(py, &[
|
||||
PyString::new(py, key).into_any().unbind(),
|
||||
size.into_pyobject(py)?.into_any().unbind(),
|
||||
mtime.into_pyobject(py)?.into_any().unbind(),
|
||||
])?;
|
||||
results_list.append(tuple)?;
|
||||
}
|
||||
dict.set_item("results", results_list)?;
|
||||
dict.set_item("truncated", truncated)?;
|
||||
|
||||
Ok(dict.into_any().unbind())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn build_object_cache(
|
||||
py: Python<'_>,
|
||||
bucket_path: &str,
|
||||
meta_root: &str,
|
||||
etag_index_path: &str,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let bucket_owned = bucket_path.to_owned();
|
||||
let meta_owned = meta_root.to_owned();
|
||||
let index_path_owned = etag_index_path.to_owned();
|
||||
|
||||
let result: (HashMap<String, String>, Vec<(String, u64, f64, Option<String>)>, bool) =
|
||||
py.detach(move || -> PyResult<(
|
||||
HashMap<String, String>,
|
||||
Vec<(String, u64, f64, Option<String>)>,
|
||||
bool,
|
||||
)> {
|
||||
let mut meta_cache: HashMap<String, String> = HashMap::new();
|
||||
let mut index_mtime: f64 = 0.0;
|
||||
let mut etag_cache_changed = false;
|
||||
|
||||
let index_p = Path::new(&index_path_owned);
|
||||
if index_p.is_file() {
|
||||
if let Ok(md) = fs::metadata(&index_path_owned) {
|
||||
index_mtime = md
|
||||
.modified()
|
||||
.map(system_time_to_epoch)
|
||||
.unwrap_or(0.0);
|
||||
}
|
||||
if let Ok(content) = fs::read_to_string(&index_path_owned) {
|
||||
if let Ok(parsed) = serde_json::from_str::<HashMap<String, String>>(&content) {
|
||||
meta_cache = parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let meta_p = Path::new(&meta_owned);
|
||||
let mut needs_rebuild = false;
|
||||
|
||||
if meta_p.is_dir() && index_mtime > 0.0 {
|
||||
fn check_newer(dir: &Path, index_mtime: f64) -> bool {
|
||||
let entries = match fs::read_dir(dir) {
|
||||
Ok(e) => e,
|
||||
Err(_) => return false,
|
||||
};
|
||||
for entry_result in entries {
|
||||
let entry = match entry_result {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let ft = match entry.file_type() {
|
||||
Ok(ft) => ft,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if ft.is_dir() && !ft.is_symlink() {
|
||||
if check_newer(&entry.path(), index_mtime) {
|
||||
return true;
|
||||
}
|
||||
} else if ft.is_file() {
|
||||
if let Some(name) = entry.file_name().to_str() {
|
||||
if name.ends_with(".meta.json") || name == "_index.json" {
|
||||
if let Ok(md) = entry.metadata() {
|
||||
let mt = md
|
||||
.modified()
|
||||
.map(system_time_to_epoch)
|
||||
.unwrap_or(0.0);
|
||||
if mt > index_mtime {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
needs_rebuild = check_newer(meta_p, index_mtime);
|
||||
} else if meta_cache.is_empty() {
|
||||
needs_rebuild = true;
|
||||
}
|
||||
|
||||
if needs_rebuild && meta_p.is_dir() {
|
||||
let meta_str = meta_owned.clone();
|
||||
let meta_len = meta_str.len() + 1;
|
||||
let mut index_files: Vec<String> = Vec::new();
|
||||
let mut legacy_meta_files: Vec<(String, String)> = Vec::new();
|
||||
|
||||
fn collect_meta(
|
||||
dir: &Path,
|
||||
meta_len: usize,
|
||||
index_files: &mut Vec<String>,
|
||||
legacy_meta_files: &mut Vec<(String, String)>,
|
||||
) {
|
||||
let entries = match fs::read_dir(dir) {
|
||||
Ok(e) => e,
|
||||
Err(_) => return,
|
||||
};
|
||||
for entry_result in entries {
|
||||
let entry = match entry_result {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let ft = match entry.file_type() {
|
||||
Ok(ft) => ft,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if ft.is_dir() && !ft.is_symlink() {
|
||||
collect_meta(&entry.path(), meta_len, index_files, legacy_meta_files);
|
||||
} else if ft.is_file() {
|
||||
if let Some(name) = entry.file_name().to_str() {
|
||||
let full = entry.path().to_string_lossy().to_string();
|
||||
if name == "_index.json" {
|
||||
index_files.push(full);
|
||||
} else if name.ends_with(".meta.json") {
|
||||
if full.len() > meta_len {
|
||||
let rel = &full[meta_len..];
|
||||
let key = if rel.len() > 10 {
|
||||
rel[..rel.len() - 10].replace('\\', "/")
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
legacy_meta_files.push((key, full));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
collect_meta(
|
||||
meta_p,
|
||||
meta_len,
|
||||
&mut index_files,
|
||||
&mut legacy_meta_files,
|
||||
);
|
||||
|
||||
meta_cache.clear();
|
||||
|
||||
for idx_path in &index_files {
|
||||
if let Ok(content) = fs::read_to_string(idx_path) {
|
||||
if let Ok(idx_data) = serde_json::from_str::<HashMap<String, Value>>(&content) {
|
||||
let rel_dir = if idx_path.len() > meta_len {
|
||||
let r = &idx_path[meta_len..];
|
||||
r.replace('\\', "/")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let dir_prefix = if rel_dir.ends_with("/_index.json") {
|
||||
&rel_dir[..rel_dir.len() - "/_index.json".len()]
|
||||
} else {
|
||||
""
|
||||
};
|
||||
for (entry_name, entry_data) in &idx_data {
|
||||
let key = if dir_prefix.is_empty() {
|
||||
entry_name.clone()
|
||||
} else {
|
||||
format!("{}/{}", dir_prefix, entry_name)
|
||||
};
|
||||
if let Some(meta_obj) = entry_data.get("metadata") {
|
||||
if let Some(etag) = meta_obj.get("__etag__") {
|
||||
if let Some(etag_str) = etag.as_str() {
|
||||
meta_cache.insert(key, etag_str.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (key, path) in &legacy_meta_files {
|
||||
if meta_cache.contains_key(key) {
|
||||
continue;
|
||||
}
|
||||
if let Ok(content) = fs::read(path) {
|
||||
if let Some(etag) = extract_etag_from_meta_bytes(&content) {
|
||||
meta_cache.insert(key.clone(), etag);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
etag_cache_changed = true;
|
||||
}
|
||||
|
||||
let bucket_p = Path::new(&bucket_owned);
|
||||
let bucket_len = bucket_owned.len() + 1;
|
||||
let mut objects: Vec<(String, u64, f64, Option<String>)> = Vec::new();
|
||||
|
||||
if bucket_p.is_dir() {
|
||||
let mut stack = vec![bucket_p.to_path_buf()];
|
||||
while let Some(current) = stack.pop() {
|
||||
let entries = match fs::read_dir(¤t) {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for entry_result in entries {
|
||||
let entry = match entry_result {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let ft = match entry.file_type() {
|
||||
Ok(ft) => ft,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if ft.is_dir() && !ft.is_symlink() {
|
||||
let full = entry.path();
|
||||
let full_str = full.to_string_lossy();
|
||||
if full_str.len() > bucket_len {
|
||||
let first_part: &str = if let Some(sep_pos) =
|
||||
full_str[bucket_len..].find(|c: char| c == '\\' || c == '/')
|
||||
{
|
||||
&full_str[bucket_len..bucket_len + sep_pos]
|
||||
} else {
|
||||
&full_str[bucket_len..]
|
||||
};
|
||||
if INTERNAL_FOLDERS.contains(&first_part) {
|
||||
continue;
|
||||
}
|
||||
} else if let Some(name) = entry.file_name().to_str() {
|
||||
if INTERNAL_FOLDERS.contains(&name) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
stack.push(full);
|
||||
} else if ft.is_file() && !ft.is_symlink() {
|
||||
let full = entry.path();
|
||||
let full_str = full.to_string_lossy();
|
||||
if full_str.len() <= bucket_len {
|
||||
continue;
|
||||
}
|
||||
let rel = &full_str[bucket_len..];
|
||||
let first_part: &str =
|
||||
if let Some(sep_pos) = rel.find(|c: char| c == '\\' || c == '/') {
|
||||
&rel[..sep_pos]
|
||||
} else {
|
||||
rel
|
||||
};
|
||||
if INTERNAL_FOLDERS.contains(&first_part) {
|
||||
continue;
|
||||
}
|
||||
let key = rel.replace('\\', "/");
|
||||
if let Ok(md) = entry.metadata() {
|
||||
let size = md.len();
|
||||
let mtime = md
|
||||
.modified()
|
||||
.map(system_time_to_epoch)
|
||||
.unwrap_or(0.0);
|
||||
let etag = meta_cache.get(&key).cloned();
|
||||
objects.push((key, size, mtime, etag));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((meta_cache, objects, etag_cache_changed))
|
||||
})?;
|
||||
|
||||
let (meta_cache, objects, etag_cache_changed) = result;
|
||||
|
||||
let dict = PyDict::new(py);
|
||||
|
||||
let cache_dict = PyDict::new(py);
|
||||
for (k, v) in &meta_cache {
|
||||
cache_dict.set_item(k, v)?;
|
||||
}
|
||||
dict.set_item("etag_cache", cache_dict)?;
|
||||
|
||||
let objects_list = PyList::empty(py);
|
||||
for (key, size, mtime, etag) in &objects {
|
||||
let etag_py: Py<PyAny> = match etag {
|
||||
Some(e) => PyString::new(py, e).into_any().unbind(),
|
||||
None => py.None(),
|
||||
};
|
||||
let tuple = PyTuple::new(py, &[
|
||||
PyString::new(py, key).into_any().unbind(),
|
||||
size.into_pyobject(py)?.into_any().unbind(),
|
||||
mtime.into_pyobject(py)?.into_any().unbind(),
|
||||
etag_py,
|
||||
])?;
|
||||
objects_list.append(tuple)?;
|
||||
}
|
||||
dict.set_item("objects", objects_list)?;
|
||||
dict.set_item("etag_cache_changed", etag_cache_changed)?;
|
||||
|
||||
Ok(dict.into_any().unbind())
|
||||
}
|
||||
112
myfsio_core/src/streaming.rs
Normal file
112
myfsio_core/src/streaming.rs
Normal file
@@ -0,0 +1,112 @@
|
||||
use md5::{Digest, Md5};
|
||||
use pyo3::exceptions::{PyIOError, PyValueError};
|
||||
use pyo3::prelude::*;
|
||||
use std::fs::{self, File};
|
||||
use std::io::{Read, Write};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_CHUNK_SIZE: usize = 262144;
|
||||
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (stream, tmp_dir, chunk_size=DEFAULT_CHUNK_SIZE))]
|
||||
pub fn stream_to_file_with_md5(
|
||||
py: Python<'_>,
|
||||
stream: &Bound<'_, PyAny>,
|
||||
tmp_dir: &str,
|
||||
chunk_size: usize,
|
||||
) -> PyResult<(String, String, u64)> {
|
||||
let chunk_size = if chunk_size == 0 {
|
||||
DEFAULT_CHUNK_SIZE
|
||||
} else {
|
||||
chunk_size
|
||||
};
|
||||
|
||||
fs::create_dir_all(tmp_dir)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to create tmp dir: {}", e)))?;
|
||||
|
||||
let tmp_name = format!("{}.tmp", Uuid::new_v4().as_hyphenated());
|
||||
let tmp_path_buf = std::path::PathBuf::from(tmp_dir).join(&tmp_name);
|
||||
let tmp_path = tmp_path_buf.to_string_lossy().into_owned();
|
||||
|
||||
let mut file = File::create(&tmp_path)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to create temp file: {}", e)))?;
|
||||
let mut hasher = Md5::new();
|
||||
let mut total_bytes: u64 = 0;
|
||||
|
||||
let result: PyResult<()> = (|| {
|
||||
loop {
|
||||
let chunk: Vec<u8> = stream.call_method1("read", (chunk_size,))?.extract()?;
|
||||
if chunk.is_empty() {
|
||||
break;
|
||||
}
|
||||
hasher.update(&chunk);
|
||||
file.write_all(&chunk)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to write: {}", e)))?;
|
||||
total_bytes += chunk.len() as u64;
|
||||
|
||||
py.check_signals()?;
|
||||
}
|
||||
file.sync_all()
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to fsync: {}", e)))?;
|
||||
Ok(())
|
||||
})();
|
||||
|
||||
if let Err(e) = result {
|
||||
drop(file);
|
||||
let _ = fs::remove_file(&tmp_path);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
drop(file);
|
||||
|
||||
let md5_hex = format!("{:x}", hasher.finalize());
|
||||
Ok((tmp_path, md5_hex, total_bytes))
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn assemble_parts_with_md5(
|
||||
py: Python<'_>,
|
||||
part_paths: Vec<String>,
|
||||
dest_path: &str,
|
||||
) -> PyResult<String> {
|
||||
if part_paths.is_empty() {
|
||||
return Err(PyValueError::new_err("No parts to assemble"));
|
||||
}
|
||||
|
||||
let dest = dest_path.to_owned();
|
||||
let parts = part_paths;
|
||||
|
||||
py.detach(move || {
|
||||
if let Some(parent) = std::path::Path::new(&dest).parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to create dest dir: {}", e)))?;
|
||||
}
|
||||
|
||||
let mut target = File::create(&dest)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to create dest file: {}", e)))?;
|
||||
let mut hasher = Md5::new();
|
||||
let mut buf = vec![0u8; 1024 * 1024];
|
||||
|
||||
for part_path in &parts {
|
||||
let mut part = File::open(part_path)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to open part {}: {}", part_path, e)))?;
|
||||
loop {
|
||||
let n = part
|
||||
.read(&mut buf)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to read part: {}", e)))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
target
|
||||
.write_all(&buf[..n])
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to write: {}", e)))?;
|
||||
}
|
||||
}
|
||||
|
||||
target.sync_all()
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to fsync: {}", e)))?;
|
||||
|
||||
Ok(format!("{:x}", hasher.finalize()))
|
||||
})
|
||||
}
|
||||
149
myfsio_core/src/validation.rs
Normal file
149
myfsio_core/src/validation.rs
Normal file
@@ -0,0 +1,149 @@
|
||||
use pyo3::prelude::*;
|
||||
use std::sync::LazyLock;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
const WINDOWS_RESERVED: &[&str] = &[
|
||||
"CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7",
|
||||
"COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8",
|
||||
"LPT9",
|
||||
];
|
||||
|
||||
const WINDOWS_ILLEGAL_CHARS: &[char] = &['<', '>', ':', '"', '/', '\\', '|', '?', '*'];
|
||||
|
||||
const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
|
||||
const SYSTEM_ROOT: &str = ".myfsio.sys";
|
||||
|
||||
static IP_REGEX: LazyLock<regex::Regex> =
|
||||
LazyLock::new(|| regex::Regex::new(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$").unwrap());
|
||||
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (object_key, max_length_bytes=1024, is_windows=false, reserved_prefixes=None))]
|
||||
pub fn validate_object_key(
|
||||
object_key: &str,
|
||||
max_length_bytes: usize,
|
||||
is_windows: bool,
|
||||
reserved_prefixes: Option<Vec<String>>,
|
||||
) -> PyResult<Option<String>> {
|
||||
if object_key.is_empty() {
|
||||
return Ok(Some("Object key required".to_string()));
|
||||
}
|
||||
|
||||
if object_key.contains('\0') {
|
||||
return Ok(Some("Object key contains null bytes".to_string()));
|
||||
}
|
||||
|
||||
let normalized: String = object_key.nfc().collect();
|
||||
|
||||
if normalized.len() > max_length_bytes {
|
||||
return Ok(Some(format!(
|
||||
"Object key exceeds maximum length of {} bytes",
|
||||
max_length_bytes
|
||||
)));
|
||||
}
|
||||
|
||||
if normalized.starts_with('/') || normalized.starts_with('\\') {
|
||||
return Ok(Some("Object key cannot start with a slash".to_string()));
|
||||
}
|
||||
|
||||
let parts: Vec<&str> = if cfg!(windows) || is_windows {
|
||||
normalized.split(['/', '\\']).collect()
|
||||
} else {
|
||||
normalized.split('/').collect()
|
||||
};
|
||||
|
||||
for part in &parts {
|
||||
if part.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if *part == ".." {
|
||||
return Ok(Some(
|
||||
"Object key contains parent directory references".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if *part == "." {
|
||||
return Ok(Some("Object key contains invalid segments".to_string()));
|
||||
}
|
||||
|
||||
if part.chars().any(|c| (c as u32) < 32) {
|
||||
return Ok(Some(
|
||||
"Object key contains control characters".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if is_windows {
|
||||
if part.chars().any(|c| WINDOWS_ILLEGAL_CHARS.contains(&c)) {
|
||||
return Ok(Some(
|
||||
"Object key contains characters not supported on Windows filesystems"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
if part.ends_with(' ') || part.ends_with('.') {
|
||||
return Ok(Some(
|
||||
"Object key segments cannot end with spaces or periods on Windows".to_string(),
|
||||
));
|
||||
}
|
||||
let trimmed = part.trim_end_matches(['.', ' ']).to_uppercase();
|
||||
if WINDOWS_RESERVED.contains(&trimmed.as_str()) {
|
||||
return Ok(Some(format!("Invalid filename segment: {}", part)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let non_empty_parts: Vec<&str> = parts.iter().filter(|p| !p.is_empty()).copied().collect();
|
||||
if let Some(top) = non_empty_parts.first() {
|
||||
if INTERNAL_FOLDERS.contains(top) || *top == SYSTEM_ROOT {
|
||||
return Ok(Some("Object key uses a reserved prefix".to_string()));
|
||||
}
|
||||
|
||||
if let Some(ref prefixes) = reserved_prefixes {
|
||||
for prefix in prefixes {
|
||||
if *top == prefix.as_str() {
|
||||
return Ok(Some("Object key uses a reserved prefix".to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn validate_bucket_name(bucket_name: &str) -> Option<String> {
|
||||
let len = bucket_name.len();
|
||||
if len < 3 || len > 63 {
|
||||
return Some("Bucket name must be between 3 and 63 characters".to_string());
|
||||
}
|
||||
|
||||
let bytes = bucket_name.as_bytes();
|
||||
if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() {
|
||||
return Some(
|
||||
"Bucket name must start and end with a lowercase letter or digit".to_string(),
|
||||
);
|
||||
}
|
||||
if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() {
|
||||
return Some(
|
||||
"Bucket name must start and end with a lowercase letter or digit".to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
for &b in bytes {
|
||||
if !b.is_ascii_lowercase() && !b.is_ascii_digit() && b != b'.' && b != b'-' {
|
||||
return Some(
|
||||
"Bucket name can only contain lowercase letters, digits, dots, and hyphens"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if bucket_name.contains("..") {
|
||||
return Some("Bucket name must not contain consecutive periods".to_string());
|
||||
}
|
||||
|
||||
if IP_REGEX.is_match(bucket_name) {
|
||||
return Some("Bucket name must not be formatted as an IP address".to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
@@ -6,7 +6,7 @@ python-dotenv>=1.2.1
|
||||
pytest>=9.0.2
|
||||
requests>=2.32.5
|
||||
boto3>=1.42.14
|
||||
waitress>=3.0.2
|
||||
granian>=2.2.0
|
||||
psutil>=7.1.3
|
||||
cryptography>=46.0.3
|
||||
defusedxml>=0.7.1
|
||||
|
||||
210
run.py
210
run.py
@@ -2,7 +2,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import warnings
|
||||
import multiprocessing
|
||||
@@ -23,6 +25,7 @@ from typing import Optional
|
||||
|
||||
from app import create_api_app, create_ui_app
|
||||
from app.config import AppConfig
|
||||
from app.iam import IamService, IamError, ALLOWED_ACTIONS, _derive_fernet_key
|
||||
|
||||
|
||||
def _server_host() -> str:
|
||||
@@ -39,24 +42,42 @@ def _is_frozen() -> bool:
|
||||
return getattr(sys, 'frozen', False) or '__compiled__' in globals()
|
||||
|
||||
|
||||
def serve_api(port: int, prod: bool = False, config: Optional[AppConfig] = None) -> None:
|
||||
app = create_api_app()
|
||||
if prod:
|
||||
from waitress import serve
|
||||
def _serve_granian(target: str, port: int, config: Optional[AppConfig] = None) -> None:
|
||||
from granian import Granian
|
||||
from granian.constants import Interfaces
|
||||
from granian.http import HTTP1Settings
|
||||
|
||||
kwargs: dict = {
|
||||
"target": target,
|
||||
"address": _server_host(),
|
||||
"port": port,
|
||||
"interface": Interfaces.WSGI,
|
||||
"factory": True,
|
||||
"workers": 1,
|
||||
}
|
||||
|
||||
if config:
|
||||
serve(
|
||||
app,
|
||||
host=_server_host(),
|
||||
port=port,
|
||||
ident="MyFSIO",
|
||||
threads=config.server_threads,
|
||||
connection_limit=config.server_connection_limit,
|
||||
backlog=config.server_backlog,
|
||||
channel_timeout=config.server_channel_timeout,
|
||||
kwargs["blocking_threads"] = config.server_threads
|
||||
kwargs["backlog"] = config.server_backlog
|
||||
kwargs["backpressure"] = config.server_connection_limit
|
||||
kwargs["http1_settings"] = HTTP1Settings(
|
||||
header_read_timeout=config.server_channel_timeout * 1000,
|
||||
max_buffer_size=config.server_max_buffer_size,
|
||||
)
|
||||
else:
|
||||
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||
kwargs["http1_settings"] = HTTP1Settings(
|
||||
max_buffer_size=1024 * 1024 * 128,
|
||||
)
|
||||
|
||||
server = Granian(**kwargs)
|
||||
server.serve()
|
||||
|
||||
|
||||
def serve_api(port: int, prod: bool = False, config: Optional[AppConfig] = None) -> None:
|
||||
if prod:
|
||||
_serve_granian("app:create_api_app", port, config)
|
||||
else:
|
||||
app = create_api_app()
|
||||
debug = _is_debug_enabled()
|
||||
if debug:
|
||||
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
||||
@@ -64,44 +85,156 @@ def serve_api(port: int, prod: bool = False, config: Optional[AppConfig] = None)
|
||||
|
||||
|
||||
def serve_ui(port: int, prod: bool = False, config: Optional[AppConfig] = None) -> None:
|
||||
app = create_ui_app()
|
||||
if prod:
|
||||
from waitress import serve
|
||||
if config:
|
||||
serve(
|
||||
app,
|
||||
host=_server_host(),
|
||||
port=port,
|
||||
ident="MyFSIO",
|
||||
threads=config.server_threads,
|
||||
connection_limit=config.server_connection_limit,
|
||||
backlog=config.server_backlog,
|
||||
channel_timeout=config.server_channel_timeout,
|
||||
)
|
||||
else:
|
||||
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||
_serve_granian("app:create_ui_app", port, config)
|
||||
else:
|
||||
app = create_ui_app()
|
||||
debug = _is_debug_enabled()
|
||||
if debug:
|
||||
warnings.warn("DEBUG MODE ENABLED - DO NOT USE IN PRODUCTION", RuntimeWarning)
|
||||
app.run(host=_server_host(), port=port, debug=debug)
|
||||
|
||||
|
||||
def reset_credentials() -> None:
|
||||
import json
|
||||
import secrets
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
config = AppConfig.from_env()
|
||||
iam_path = config.iam_config_path
|
||||
encryption_key = config.secret_key
|
||||
|
||||
access_key = os.environ.get("ADMIN_ACCESS_KEY", "").strip() or secrets.token_hex(12)
|
||||
secret_key = os.environ.get("ADMIN_SECRET_KEY", "").strip() or secrets.token_urlsafe(32)
|
||||
custom_keys = bool(os.environ.get("ADMIN_ACCESS_KEY", "").strip())
|
||||
|
||||
fernet = Fernet(_derive_fernet_key(encryption_key)) if encryption_key else None
|
||||
|
||||
raw_config = None
|
||||
if iam_path.exists():
|
||||
try:
|
||||
raw_bytes = iam_path.read_bytes()
|
||||
from app.iam import _IAM_ENCRYPTED_PREFIX
|
||||
if raw_bytes.startswith(_IAM_ENCRYPTED_PREFIX):
|
||||
if fernet:
|
||||
try:
|
||||
content = fernet.decrypt(raw_bytes[len(_IAM_ENCRYPTED_PREFIX):]).decode("utf-8")
|
||||
raw_config = json.loads(content)
|
||||
except Exception:
|
||||
print("WARNING: Could not decrypt existing IAM config. Creating fresh config.")
|
||||
else:
|
||||
print("WARNING: IAM config is encrypted but no SECRET_KEY available. Creating fresh config.")
|
||||
else:
|
||||
try:
|
||||
raw_config = json.loads(raw_bytes.decode("utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
print("WARNING: Existing IAM config is corrupted. Creating fresh config.")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if raw_config and raw_config.get("users"):
|
||||
is_v2 = raw_config.get("version", 1) >= 2
|
||||
admin_user = None
|
||||
for user in raw_config["users"]:
|
||||
policies = user.get("policies", [])
|
||||
for p in policies:
|
||||
actions = p.get("actions", [])
|
||||
if "iam:*" in actions or "*" in actions:
|
||||
admin_user = user
|
||||
break
|
||||
if admin_user:
|
||||
break
|
||||
if not admin_user:
|
||||
admin_user = raw_config["users"][0]
|
||||
|
||||
if is_v2:
|
||||
admin_keys = admin_user.get("access_keys", [])
|
||||
if admin_keys:
|
||||
admin_keys[0]["access_key"] = access_key
|
||||
admin_keys[0]["secret_key"] = secret_key
|
||||
else:
|
||||
from datetime import datetime as _dt, timezone as _tz
|
||||
admin_user["access_keys"] = [{
|
||||
"access_key": access_key,
|
||||
"secret_key": secret_key,
|
||||
"status": "active",
|
||||
"created_at": _dt.now(_tz.utc).isoformat(),
|
||||
}]
|
||||
else:
|
||||
admin_user["access_key"] = access_key
|
||||
admin_user["secret_key"] = secret_key
|
||||
else:
|
||||
from datetime import datetime as _dt, timezone as _tz
|
||||
raw_config = {
|
||||
"version": 2,
|
||||
"users": [
|
||||
{
|
||||
"user_id": f"u-{secrets.token_hex(8)}",
|
||||
"display_name": "Local Admin",
|
||||
"enabled": True,
|
||||
"access_keys": [
|
||||
{
|
||||
"access_key": access_key,
|
||||
"secret_key": secret_key,
|
||||
"status": "active",
|
||||
"created_at": _dt.now(_tz.utc).isoformat(),
|
||||
}
|
||||
],
|
||||
"policies": [
|
||||
{"bucket": "*", "actions": list(ALLOWED_ACTIONS)}
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
json_text = json.dumps(raw_config, indent=2)
|
||||
iam_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp_path = iam_path.with_suffix(".json.tmp")
|
||||
if fernet:
|
||||
from app.iam import _IAM_ENCRYPTED_PREFIX
|
||||
encrypted = fernet.encrypt(json_text.encode("utf-8"))
|
||||
temp_path.write_bytes(_IAM_ENCRYPTED_PREFIX + encrypted)
|
||||
else:
|
||||
temp_path.write_text(json_text, encoding="utf-8")
|
||||
temp_path.replace(iam_path)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print("MYFSIO - ADMIN CREDENTIALS RESET")
|
||||
print(f"{'='*60}")
|
||||
if custom_keys:
|
||||
print(f"Access Key: {access_key} (from ADMIN_ACCESS_KEY)")
|
||||
print(f"Secret Key: {'(from ADMIN_SECRET_KEY)' if os.environ.get('ADMIN_SECRET_KEY', '').strip() else secret_key}")
|
||||
else:
|
||||
print(f"Access Key: {access_key}")
|
||||
print(f"Secret Key: {secret_key}")
|
||||
print(f"{'='*60}")
|
||||
if fernet:
|
||||
print("IAM config saved (encrypted).")
|
||||
else:
|
||||
print(f"IAM config saved to: {iam_path}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
multiprocessing.freeze_support()
|
||||
if _is_frozen():
|
||||
multiprocessing.set_start_method("spawn", force=True)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Run the S3 clone services.")
|
||||
parser.add_argument("--mode", choices=["api", "ui", "both"], default="both")
|
||||
parser.add_argument("--mode", choices=["api", "ui", "both", "reset-cred"], default="both")
|
||||
parser.add_argument("--api-port", type=int, default=5000)
|
||||
parser.add_argument("--ui-port", type=int, default=5100)
|
||||
parser.add_argument("--prod", action="store_true", help="Run in production mode using Waitress")
|
||||
parser.add_argument("--prod", action="store_true", help="Run in production mode using Granian")
|
||||
parser.add_argument("--dev", action="store_true", help="Force development mode (Flask dev server)")
|
||||
parser.add_argument("--check-config", action="store_true", help="Validate configuration and exit")
|
||||
parser.add_argument("--show-config", action="store_true", help="Show configuration summary and exit")
|
||||
parser.add_argument("--reset-cred", action="store_true", help="Reset admin credentials and exit")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.reset_cred or args.mode == "reset-cred":
|
||||
reset_credentials()
|
||||
sys.exit(0)
|
||||
|
||||
if args.check_config or args.show_config:
|
||||
config = AppConfig.from_env()
|
||||
config.print_startup_summary()
|
||||
@@ -134,7 +267,7 @@ if __name__ == "__main__":
|
||||
pass
|
||||
|
||||
if prod_mode:
|
||||
print("Running in production mode (Waitress)")
|
||||
print("Running in production mode (Granian)")
|
||||
issues = config.validate_and_report()
|
||||
critical_issues = [i for i in issues if i.startswith("CRITICAL:")]
|
||||
if critical_issues:
|
||||
@@ -147,11 +280,22 @@ if __name__ == "__main__":
|
||||
|
||||
if args.mode in {"api", "both"}:
|
||||
print(f"Starting API server on port {args.api_port}...")
|
||||
api_proc = Process(target=serve_api, args=(args.api_port, prod_mode, config), daemon=True)
|
||||
api_proc = Process(target=serve_api, args=(args.api_port, prod_mode, config))
|
||||
api_proc.start()
|
||||
else:
|
||||
api_proc = None
|
||||
|
||||
def _cleanup_api():
|
||||
if api_proc and api_proc.is_alive():
|
||||
api_proc.terminate()
|
||||
api_proc.join(timeout=5)
|
||||
if api_proc.is_alive():
|
||||
api_proc.kill()
|
||||
|
||||
if api_proc:
|
||||
atexit.register(_cleanup_api)
|
||||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
|
||||
|
||||
if args.mode in {"ui", "both"}:
|
||||
print(f"Starting UI server on port {args.ui_port}...")
|
||||
serve_ui(args.ui_port, prod_mode, config)
|
||||
|
||||
@@ -379,29 +379,25 @@ if [[ "$SKIP_SYSTEMD" != true ]]; then
|
||||
echo " ---------------"
|
||||
if systemctl is-active --quiet myfsio; then
|
||||
echo " [OK] MyFSIO is running"
|
||||
|
||||
IAM_FILE="$DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
if [[ -f "$IAM_FILE" ]]; then
|
||||
echo ""
|
||||
echo " ============================================"
|
||||
echo " ADMIN CREDENTIALS (save these securely!)"
|
||||
echo " ============================================"
|
||||
if command -v jq &>/dev/null; then
|
||||
ACCESS_KEY=$(jq -r '.users[0].access_key' "$IAM_FILE" 2>/dev/null)
|
||||
SECRET_KEY=$(jq -r '.users[0].secret_key' "$IAM_FILE" 2>/dev/null)
|
||||
else
|
||||
ACCESS_KEY=$(grep -o '"access_key"[[:space:]]*:[[:space:]]*"[^"]*"' "$IAM_FILE" | head -1 | sed 's/.*"\([^"]*\)"$/\1/')
|
||||
SECRET_KEY=$(grep -o '"secret_key"[[:space:]]*:[[:space:]]*"[^"]*"' "$IAM_FILE" | head -1 | sed 's/.*"\([^"]*\)"$/\1/')
|
||||
fi
|
||||
if [[ -n "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then
|
||||
CRED_OUTPUT=$(journalctl -u myfsio --no-pager -n 50 2>/dev/null | grep -A 5 "FIRST RUN - ADMIN CREDENTIALS")
|
||||
ACCESS_KEY=$(echo "$CRED_OUTPUT" | grep "Access Key:" | head -1 | sed 's/.*Access Key: //' | awk '{print $1}')
|
||||
SECRET_KEY=$(echo "$CRED_OUTPUT" | grep "Secret Key:" | head -1 | sed 's/.*Secret Key: //' | awk '{print $1}')
|
||||
if [[ -n "$ACCESS_KEY" && "$ACCESS_KEY" != *"from"* && -n "$SECRET_KEY" && "$SECRET_KEY" != *"from"* ]]; then
|
||||
echo " Access Key: $ACCESS_KEY"
|
||||
echo " Secret Key: $SECRET_KEY"
|
||||
else
|
||||
echo " [!] Could not parse credentials from $IAM_FILE"
|
||||
echo " Check the file manually or view service logs."
|
||||
echo " [!] Could not extract credentials from service logs."
|
||||
echo " Check startup output: journalctl -u myfsio --no-pager | grep -A 5 'ADMIN CREDENTIALS'"
|
||||
echo " Or reset credentials: $INSTALL_DIR/myfsio reset-cred"
|
||||
fi
|
||||
echo " ============================================"
|
||||
fi
|
||||
echo ""
|
||||
echo " NOTE: The IAM config file is encrypted at rest."
|
||||
echo " Credentials are only shown on first run or after reset."
|
||||
else
|
||||
echo " [WARNING] MyFSIO may not have started correctly"
|
||||
echo " Check logs with: journalctl -u myfsio -f"
|
||||
@@ -427,12 +423,13 @@ echo " API: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "local
|
||||
echo " UI: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost"):$UI_PORT/ui"
|
||||
echo ""
|
||||
echo "Credentials:"
|
||||
echo " Admin credentials were shown above (if service was started)."
|
||||
echo " You can also find them in: $DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
echo " Admin credentials are shown on first service start (see above)."
|
||||
echo " The IAM config is encrypted at rest and cannot be read directly."
|
||||
echo " To reset credentials: $INSTALL_DIR/myfsio reset-cred"
|
||||
echo ""
|
||||
echo "Configuration Files:"
|
||||
echo " Environment: $INSTALL_DIR/myfsio.env"
|
||||
echo " IAM Users: $DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
echo " IAM Users: $DATA_DIR/.myfsio.sys/config/iam.json (encrypted)"
|
||||
echo " Bucket Policies: $DATA_DIR/.myfsio.sys/config/bucket_policies.json"
|
||||
echo " Secret Key: $DATA_DIR/.myfsio.sys/config/.secret (auto-generated)"
|
||||
echo ""
|
||||
|
||||
@@ -230,11 +230,14 @@ if [[ "$KEEP_DATA" == true ]]; then
|
||||
echo ""
|
||||
echo "Preserved files include:"
|
||||
echo " - All buckets and objects"
|
||||
echo " - IAM configuration: $DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
echo " - IAM configuration: $DATA_DIR/.myfsio.sys/config/iam.json (encrypted at rest)"
|
||||
echo " - Bucket policies: $DATA_DIR/.myfsio.sys/config/bucket_policies.json"
|
||||
echo " - Secret key: $DATA_DIR/.myfsio.sys/config/.secret"
|
||||
echo " - Encryption keys: $DATA_DIR/.myfsio.sys/keys/ (if encryption was enabled)"
|
||||
echo ""
|
||||
echo "NOTE: The IAM config is encrypted and requires the SECRET_KEY to read."
|
||||
echo " Keep the .secret file intact for reinstallation."
|
||||
echo ""
|
||||
echo "To reinstall MyFSIO with existing data:"
|
||||
echo " ./install.sh --data-dir $DATA_DIR"
|
||||
echo ""
|
||||
|
||||
@@ -15,6 +15,12 @@
|
||||
--myfsio-hover-bg: rgba(59, 130, 246, 0.12);
|
||||
--myfsio-accent: #3b82f6;
|
||||
--myfsio-accent-hover: #2563eb;
|
||||
--myfsio-tag-key-bg: #e0e7ff;
|
||||
--myfsio-tag-key-text: #3730a3;
|
||||
--myfsio-tag-value-bg: #f0f1fa;
|
||||
--myfsio-tag-value-text: #4338ca;
|
||||
--myfsio-tag-border: #c7d2fe;
|
||||
--myfsio-tag-delete-hover: #ef4444;
|
||||
}
|
||||
|
||||
[data-theme='dark'] {
|
||||
@@ -34,6 +40,12 @@
|
||||
--myfsio-hover-bg: rgba(59, 130, 246, 0.2);
|
||||
--myfsio-accent: #60a5fa;
|
||||
--myfsio-accent-hover: #3b82f6;
|
||||
--myfsio-tag-key-bg: #312e81;
|
||||
--myfsio-tag-key-text: #c7d2fe;
|
||||
--myfsio-tag-value-bg: #1e1b4b;
|
||||
--myfsio-tag-value-text: #a5b4fc;
|
||||
--myfsio-tag-border: #4338ca;
|
||||
--myfsio-tag-delete-hover: #f87171;
|
||||
}
|
||||
|
||||
[data-theme='dark'] body,
|
||||
@@ -1151,17 +1163,104 @@ html.sidebar-will-collapse .sidebar-user {
|
||||
}
|
||||
|
||||
.iam-user-card {
|
||||
border: 1px solid var(--myfsio-card-border);
|
||||
border-radius: 0.75rem;
|
||||
transition: box-shadow 0.2s ease, transform 0.2s ease;
|
||||
position: relative;
|
||||
border: 1px solid var(--myfsio-card-border) !important;
|
||||
border-radius: 1rem !important;
|
||||
overflow: visible;
|
||||
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
}
|
||||
|
||||
.iam-user-card:hover {
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 8px 24px -4px rgba(0, 0, 0, 0.12), 0 4px 8px -4px rgba(0, 0, 0, 0.08);
|
||||
border-color: var(--myfsio-accent) !important;
|
||||
}
|
||||
|
||||
[data-theme='dark'] .iam-user-card:hover {
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
|
||||
box-shadow: 0 8px 24px -4px rgba(0, 0, 0, 0.4), 0 4px 8px -4px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
|
||||
.iam-role-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 0.25em 0.65em;
|
||||
border-radius: 999px;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.03em;
|
||||
}
|
||||
|
||||
.iam-role-admin {
|
||||
background: rgba(245, 158, 11, 0.15);
|
||||
color: #d97706;
|
||||
}
|
||||
|
||||
[data-theme='dark'] .iam-role-admin {
|
||||
background: rgba(245, 158, 11, 0.25);
|
||||
color: #fbbf24;
|
||||
}
|
||||
|
||||
.iam-role-user {
|
||||
background: rgba(59, 130, 246, 0.12);
|
||||
color: #2563eb;
|
||||
}
|
||||
|
||||
[data-theme='dark'] .iam-role-user {
|
||||
background: rgba(59, 130, 246, 0.2);
|
||||
color: #60a5fa;
|
||||
}
|
||||
|
||||
.iam-perm-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
padding: 0.3em 0.6em;
|
||||
border-radius: 999px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
background: rgba(59, 130, 246, 0.08);
|
||||
color: var(--myfsio-text);
|
||||
border: 1px solid rgba(59, 130, 246, 0.15);
|
||||
}
|
||||
|
||||
[data-theme='dark'] .iam-perm-badge {
|
||||
background: rgba(59, 130, 246, 0.15);
|
||||
border-color: rgba(59, 130, 246, 0.25);
|
||||
}
|
||||
|
||||
.iam-copy-key {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 22px;
|
||||
height: 22px;
|
||||
padding: 0;
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: var(--myfsio-muted);
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.iam-copy-key:hover {
|
||||
background: var(--myfsio-hover-bg);
|
||||
color: var(--myfsio-text);
|
||||
}
|
||||
|
||||
.iam-no-results {
|
||||
text-align: center;
|
||||
padding: 2rem 1rem;
|
||||
color: var(--myfsio-muted);
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.iam-user-card:hover {
|
||||
transform: none;
|
||||
}
|
||||
}
|
||||
|
||||
.user-avatar-lg {
|
||||
@@ -1288,6 +1387,20 @@ html.sidebar-will-collapse .sidebar-user {
|
||||
padding: 2rem 1rem;
|
||||
}
|
||||
|
||||
#preview-text {
|
||||
padding: 1rem 1.125rem;
|
||||
max-height: 360px;
|
||||
overflow: auto;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-family: 'SFMono-Regular', 'Menlo', 'Consolas', 'Liberation Mono', monospace;
|
||||
font-size: .8rem;
|
||||
line-height: 1.6;
|
||||
tab-size: 4;
|
||||
color: var(--myfsio-text);
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.upload-progress-stack {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -2542,7 +2655,7 @@ pre code {
|
||||
}
|
||||
|
||||
.objects-table-container {
|
||||
max-height: none;
|
||||
max-height: 60vh;
|
||||
}
|
||||
|
||||
.preview-card {
|
||||
@@ -2805,6 +2918,195 @@ body:has(.login-card) .main-wrapper {
|
||||
padding-top: 0 !important;
|
||||
}
|
||||
|
||||
.context-menu {
|
||||
position: fixed;
|
||||
z-index: 1060;
|
||||
min-width: 180px;
|
||||
background: var(--myfsio-card-bg);
|
||||
border: 1px solid var(--myfsio-card-border);
|
||||
border-radius: 0.5rem;
|
||||
box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.15), 0 8px 10px -6px rgba(0, 0, 0, 0.1);
|
||||
padding: 0.25rem 0;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
[data-theme='dark'] .context-menu {
|
||||
box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.4), 0 8px 10px -6px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
.context-menu-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.625rem;
|
||||
padding: 0.5rem 0.875rem;
|
||||
color: var(--myfsio-text);
|
||||
cursor: pointer;
|
||||
transition: background-color 0.1s ease;
|
||||
border: none;
|
||||
background: none;
|
||||
width: 100%;
|
||||
text-align: left;
|
||||
font-size: inherit;
|
||||
}
|
||||
|
||||
.context-menu-item:hover {
|
||||
background-color: var(--myfsio-hover-bg);
|
||||
}
|
||||
|
||||
.context-menu-item.text-danger:hover {
|
||||
background-color: rgba(239, 68, 68, 0.1);
|
||||
}
|
||||
|
||||
.context-menu-divider {
|
||||
height: 1px;
|
||||
background: var(--myfsio-card-border);
|
||||
margin: 0.25rem 0;
|
||||
}
|
||||
|
||||
.context-menu-shortcut {
|
||||
margin-left: auto;
|
||||
font-size: 0.75rem;
|
||||
color: var(--myfsio-muted);
|
||||
}
|
||||
|
||||
.kbd-shortcuts-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.kbd-shortcuts-list .shortcut-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 0.375rem 0;
|
||||
}
|
||||
|
||||
.kbd-shortcuts-list kbd {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-width: 1.75rem;
|
||||
padding: 0.2rem 0.5rem;
|
||||
font-family: inherit;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
background: var(--myfsio-preview-bg);
|
||||
border: 1px solid var(--myfsio-card-border);
|
||||
border-radius: 0.25rem;
|
||||
box-shadow: 0 1px 0 1px rgba(0, 0, 0, 0.05);
|
||||
color: var(--myfsio-text);
|
||||
}
|
||||
|
||||
[data-theme='dark'] .kbd-shortcuts-list kbd {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
box-shadow: 0 1px 0 1px rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
|
||||
.sort-dropdown .dropdown-item.active,
|
||||
.sort-dropdown .dropdown-item:active {
|
||||
background-color: var(--myfsio-hover-bg);
|
||||
color: var(--myfsio-text);
|
||||
}
|
||||
|
||||
.sort-dropdown .dropdown-item {
|
||||
font-size: 0.875rem;
|
||||
padding: 0.375rem 1rem;
|
||||
}
|
||||
|
||||
.tag-pill {
|
||||
display: inline-flex;
|
||||
border-radius: 9999px;
|
||||
border: 1px solid var(--myfsio-tag-border);
|
||||
overflow: hidden;
|
||||
font-size: 0.75rem;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.tag-pill-key {
|
||||
padding: 0.3rem 0.5rem;
|
||||
background: var(--myfsio-tag-key-bg);
|
||||
color: var(--myfsio-tag-key-text);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.tag-pill-value {
|
||||
padding: 0.3rem 0.5rem;
|
||||
background: var(--myfsio-tag-value-bg);
|
||||
color: var(--myfsio-tag-value-text);
|
||||
font-weight: 400;
|
||||
}
|
||||
|
||||
.tag-editor-card {
|
||||
background: var(--myfsio-preview-bg);
|
||||
border-radius: 0.5rem;
|
||||
padding: 0.75rem;
|
||||
}
|
||||
|
||||
.tag-editor-header,
|
||||
.tag-editor-row {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr 28px;
|
||||
gap: 0.5rem;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.tag-editor-header {
|
||||
padding-bottom: 0.375rem;
|
||||
border-bottom: 1px solid var(--myfsio-card-border);
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.tag-editor-header span {
|
||||
font-size: 0.7rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
color: var(--myfsio-muted);
|
||||
letter-spacing: 0.05em;
|
||||
}
|
||||
|
||||
.tag-editor-row {
|
||||
margin-bottom: 0.375rem;
|
||||
}
|
||||
|
||||
.tag-editor-delete {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: var(--myfsio-muted);
|
||||
border-radius: 0.375rem;
|
||||
cursor: pointer;
|
||||
transition: color 0.15s, background 0.15s;
|
||||
}
|
||||
|
||||
.tag-editor-delete:hover {
|
||||
color: var(--myfsio-tag-delete-hover);
|
||||
background: rgba(239, 68, 68, 0.1);
|
||||
}
|
||||
|
||||
.tag-editor-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-top: 0.75rem;
|
||||
padding-top: 0.5rem;
|
||||
border-top: 1px solid var(--myfsio-card-border);
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
animation-duration: 0.01ms !important;
|
||||
animation-iteration-count: 1 !important;
|
||||
transition-duration: 0.01ms !important;
|
||||
}
|
||||
}
|
||||
|
||||
@media print {
|
||||
.sidebar,
|
||||
.mobile-header {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,8 @@ window.BucketDetailUpload = (function() {
|
||||
|
||||
const MULTIPART_THRESHOLD = 8 * 1024 * 1024;
|
||||
const CHUNK_SIZE = 8 * 1024 * 1024;
|
||||
const MAX_PART_RETRIES = 3;
|
||||
const RETRY_BASE_DELAY_MS = 1000;
|
||||
|
||||
let state = {
|
||||
isUploading: false,
|
||||
@@ -204,6 +206,67 @@ window.BucketDetailUpload = (function() {
|
||||
}
|
||||
}
|
||||
|
||||
function uploadPartXHR(url, chunk, csrfToken, baseBytes, fileSize, progressItem, partNumber, totalParts) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('PUT', url, true);
|
||||
xhr.setRequestHeader('X-CSRFToken', csrfToken || '');
|
||||
|
||||
xhr.upload.addEventListener('progress', (e) => {
|
||||
if (e.lengthComputable) {
|
||||
updateProgressItem(progressItem, {
|
||||
status: `Part ${partNumber}/${totalParts}`,
|
||||
loaded: baseBytes + e.loaded,
|
||||
total: fileSize
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('load', () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
try {
|
||||
resolve(JSON.parse(xhr.responseText));
|
||||
} catch {
|
||||
reject(new Error(`Part ${partNumber}: invalid response`));
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
const data = JSON.parse(xhr.responseText);
|
||||
reject(new Error(data.error || `Part ${partNumber} failed (${xhr.status})`));
|
||||
} catch {
|
||||
reject(new Error(`Part ${partNumber} failed (${xhr.status})`));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
xhr.addEventListener('error', () => reject(new Error(`Part ${partNumber}: network error`)));
|
||||
xhr.addEventListener('abort', () => reject(new Error(`Part ${partNumber}: aborted`)));
|
||||
|
||||
xhr.send(chunk);
|
||||
});
|
||||
}
|
||||
|
||||
async function uploadPartWithRetry(url, chunk, csrfToken, baseBytes, fileSize, progressItem, partNumber, totalParts) {
|
||||
let lastError;
|
||||
for (let attempt = 0; attempt <= MAX_PART_RETRIES; attempt++) {
|
||||
try {
|
||||
return await uploadPartXHR(url, chunk, csrfToken, baseBytes, fileSize, progressItem, partNumber, totalParts);
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
if (attempt < MAX_PART_RETRIES) {
|
||||
const delay = RETRY_BASE_DELAY_MS * Math.pow(2, attempt);
|
||||
updateProgressItem(progressItem, {
|
||||
status: `Part ${partNumber}/${totalParts} retry ${attempt + 1}/${MAX_PART_RETRIES}...`,
|
||||
loaded: baseBytes,
|
||||
total: fileSize
|
||||
});
|
||||
await new Promise(r => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
async function uploadMultipart(file, objectKey, metadata, progressItem, urls) {
|
||||
const csrfToken = document.querySelector('input[name="csrf_token"]')?.value;
|
||||
|
||||
@@ -233,26 +296,14 @@ window.BucketDetailUpload = (function() {
|
||||
const end = Math.min(start + CHUNK_SIZE, file.size);
|
||||
const chunk = file.slice(start, end);
|
||||
|
||||
updateProgressItem(progressItem, {
|
||||
status: `Part ${partNumber}/${totalParts}`,
|
||||
loaded: uploadedBytes,
|
||||
total: file.size
|
||||
});
|
||||
const partData = await uploadPartWithRetry(
|
||||
`${partUrl}?partNumber=${partNumber}`,
|
||||
chunk, csrfToken, uploadedBytes, file.size,
|
||||
progressItem, partNumber, totalParts
|
||||
);
|
||||
|
||||
const partResp = await fetch(`${partUrl}?partNumber=${partNumber}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'X-CSRFToken': csrfToken || '' },
|
||||
body: chunk
|
||||
});
|
||||
|
||||
if (!partResp.ok) {
|
||||
const err = await partResp.json().catch(() => ({}));
|
||||
throw new Error(err.error || `Part ${partNumber} failed`);
|
||||
}
|
||||
|
||||
const partData = await partResp.json();
|
||||
parts.push({ part_number: partNumber, etag: partData.etag });
|
||||
uploadedBytes += chunk.size;
|
||||
uploadedBytes += (end - start);
|
||||
|
||||
updateProgressItem(progressItem, {
|
||||
loaded: uploadedBytes,
|
||||
|
||||
@@ -78,7 +78,7 @@ window.ConnectionsManagement = (function() {
|
||||
|
||||
try {
|
||||
var controller = new AbortController();
|
||||
var timeoutId = setTimeout(function() { controller.abort(); }, 15000);
|
||||
var timeoutId = setTimeout(function() { controller.abort(); }, 10000);
|
||||
|
||||
var response = await fetch(endpoints.healthTemplate.replace('CONNECTION_ID', connectionId), {
|
||||
signal: controller.signal
|
||||
@@ -147,7 +147,7 @@ window.ConnectionsManagement = (function() {
|
||||
'<button type="button" class="btn btn-outline-secondary" data-bs-toggle="modal" data-bs-target="#editConnectionModal" ' +
|
||||
'data-id="' + window.UICore.escapeHtml(conn.id) + '" data-name="' + window.UICore.escapeHtml(conn.name) + '" ' +
|
||||
'data-endpoint="' + window.UICore.escapeHtml(conn.endpoint_url) + '" data-region="' + window.UICore.escapeHtml(conn.region) + '" ' +
|
||||
'data-access="' + window.UICore.escapeHtml(conn.access_key) + '" data-secret="' + window.UICore.escapeHtml(conn.secret_key || '') + '" title="Edit connection">' +
|
||||
'data-access="' + window.UICore.escapeHtml(conn.access_key) + '" title="Edit connection">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">' +
|
||||
'<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/></svg></button>' +
|
||||
'<button type="button" class="btn btn-outline-danger" data-bs-toggle="modal" data-bs-target="#deleteConnectionModal" ' +
|
||||
@@ -185,7 +185,9 @@ window.ConnectionsManagement = (function() {
|
||||
document.getElementById('edit_endpoint_url').value = button.getAttribute('data-endpoint') || '';
|
||||
document.getElementById('edit_region').value = button.getAttribute('data-region') || '';
|
||||
document.getElementById('edit_access_key').value = button.getAttribute('data-access') || '';
|
||||
document.getElementById('edit_secret_key').value = button.getAttribute('data-secret') || '';
|
||||
document.getElementById('edit_secret_key').value = '';
|
||||
document.getElementById('edit_secret_key').placeholder = '(unchanged — leave blank to keep current)';
|
||||
document.getElementById('edit_secret_key').required = false;
|
||||
document.getElementById('editTestResult').innerHTML = '';
|
||||
|
||||
var form = document.getElementById('editConnectionForm');
|
||||
@@ -288,9 +290,6 @@ window.ConnectionsManagement = (function() {
|
||||
editBtn.setAttribute('data-endpoint', data.connection.endpoint_url);
|
||||
editBtn.setAttribute('data-region', data.connection.region);
|
||||
editBtn.setAttribute('data-access', data.connection.access_key);
|
||||
if (data.connection.secret_key) {
|
||||
editBtn.setAttribute('data-secret', data.connection.secret_key);
|
||||
}
|
||||
}
|
||||
|
||||
var deleteBtn = row.querySelector('[data-bs-target="#deleteConnectionModal"]');
|
||||
|
||||
@@ -11,16 +11,53 @@ window.IAMManagement = (function() {
|
||||
var editUserModal = null;
|
||||
var deleteUserModal = null;
|
||||
var rotateSecretModal = null;
|
||||
var expiryModal = null;
|
||||
var currentRotateKey = null;
|
||||
var currentEditKey = null;
|
||||
var currentDeleteKey = null;
|
||||
var currentExpiryKey = null;
|
||||
|
||||
var ALL_S3_ACTIONS = [
|
||||
'list', 'read', 'write', 'delete', 'share', 'policy',
|
||||
'replication', 'lifecycle', 'cors',
|
||||
'create_bucket', 'delete_bucket',
|
||||
'versioning', 'tagging', 'encryption', 'quota',
|
||||
'object_lock', 'notification', 'logging', 'website'
|
||||
];
|
||||
|
||||
var policyTemplates = {
|
||||
full: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'share', 'policy', 'replication', 'lifecycle', 'cors', 'iam:*'] }],
|
||||
full: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'share', 'policy', 'create_bucket', 'delete_bucket', 'replication', 'lifecycle', 'cors', 'versioning', 'tagging', 'encryption', 'quota', 'object_lock', 'notification', 'logging', 'website', 'iam:*'] }],
|
||||
readonly: [{ bucket: '*', actions: ['list', 'read'] }],
|
||||
writer: [{ bucket: '*', actions: ['list', 'read', 'write'] }]
|
||||
writer: [{ bucket: '*', actions: ['list', 'read', 'write'] }],
|
||||
operator: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'create_bucket', 'delete_bucket'] }],
|
||||
bucketadmin: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'share', 'policy', 'create_bucket', 'delete_bucket', 'versioning', 'tagging', 'encryption', 'cors', 'lifecycle', 'quota', 'object_lock', 'notification', 'logging', 'website', 'replication'] }]
|
||||
};
|
||||
|
||||
function isAdminUser(policies) {
|
||||
if (!policies || !policies.length) return false;
|
||||
return policies.some(function(p) {
|
||||
return p.actions && (p.actions.indexOf('iam:*') >= 0 || p.actions.indexOf('*') >= 0);
|
||||
});
|
||||
}
|
||||
|
||||
function getPermissionLevel(actions) {
|
||||
if (!actions || !actions.length) return 'Custom (0)';
|
||||
if (actions.indexOf('*') >= 0) return 'Full Access';
|
||||
if (actions.length >= ALL_S3_ACTIONS.length) {
|
||||
var hasAll = ALL_S3_ACTIONS.every(function(a) { return actions.indexOf(a) >= 0; });
|
||||
if (hasAll) return 'Full Access';
|
||||
}
|
||||
var has = function(a) { return actions.indexOf(a) >= 0; };
|
||||
if (has('list') && has('read') && has('write') && has('delete')) return 'Read + Write + Delete';
|
||||
if (has('list') && has('read') && has('write')) return 'Read + Write';
|
||||
if (has('list') && has('read')) return 'Read Only';
|
||||
return 'Custom (' + actions.length + ')';
|
||||
}
|
||||
|
||||
function getBucketLabel(bucket) {
|
||||
return bucket === '*' ? 'All Buckets' : bucket;
|
||||
}
|
||||
|
||||
function init(config) {
|
||||
users = config.users || [];
|
||||
currentUserKey = config.currentUserKey || null;
|
||||
@@ -38,7 +75,10 @@ window.IAMManagement = (function() {
|
||||
setupEditUserModal();
|
||||
setupDeleteUserModal();
|
||||
setupRotateSecretModal();
|
||||
setupExpiryModal();
|
||||
setupFormHandlers();
|
||||
setupSearch();
|
||||
setupCopyAccessKeyButtons();
|
||||
}
|
||||
|
||||
function initModals() {
|
||||
@@ -46,11 +86,13 @@ window.IAMManagement = (function() {
|
||||
var editModalEl = document.getElementById('editUserModal');
|
||||
var deleteModalEl = document.getElementById('deleteUserModal');
|
||||
var rotateModalEl = document.getElementById('rotateSecretModal');
|
||||
var expiryModalEl = document.getElementById('expiryModal');
|
||||
|
||||
if (policyModalEl) policyModal = new bootstrap.Modal(policyModalEl);
|
||||
if (editModalEl) editUserModal = new bootstrap.Modal(editModalEl);
|
||||
if (deleteModalEl) deleteUserModal = new bootstrap.Modal(deleteModalEl);
|
||||
if (rotateModalEl) rotateSecretModal = new bootstrap.Modal(rotateModalEl);
|
||||
if (expiryModalEl) expiryModal = new bootstrap.Modal(expiryModalEl);
|
||||
}
|
||||
|
||||
function setupJsonAutoIndent() {
|
||||
@@ -68,6 +110,15 @@ window.IAMManagement = (function() {
|
||||
});
|
||||
});
|
||||
|
||||
var accessKeyCopyButton = document.querySelector('[data-access-key-copy]');
|
||||
if (accessKeyCopyButton) {
|
||||
accessKeyCopyButton.addEventListener('click', async function() {
|
||||
var accessKeyInput = document.getElementById('disclosedAccessKeyValue');
|
||||
if (!accessKeyInput) return;
|
||||
await window.UICore.copyToClipboard(accessKeyInput.value, accessKeyCopyButton, 'Copy');
|
||||
});
|
||||
}
|
||||
|
||||
var secretCopyButton = document.querySelector('[data-secret-copy]');
|
||||
if (secretCopyButton) {
|
||||
secretCopyButton.addEventListener('click', async function() {
|
||||
@@ -114,6 +165,22 @@ window.IAMManagement = (function() {
|
||||
});
|
||||
}
|
||||
|
||||
function generateSecureHex(byteCount) {
|
||||
var arr = new Uint8Array(byteCount);
|
||||
crypto.getRandomValues(arr);
|
||||
return Array.from(arr).map(function(b) { return b.toString(16).padStart(2, '0'); }).join('');
|
||||
}
|
||||
|
||||
function generateSecureBase64(byteCount) {
|
||||
var arr = new Uint8Array(byteCount);
|
||||
crypto.getRandomValues(arr);
|
||||
var binary = '';
|
||||
for (var i = 0; i < arr.length; i++) {
|
||||
binary += String.fromCharCode(arr[i]);
|
||||
}
|
||||
return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
|
||||
}
|
||||
|
||||
function setupCreateUserModal() {
|
||||
var createUserPoliciesEl = document.getElementById('createUserPolicies');
|
||||
|
||||
@@ -122,6 +189,22 @@ window.IAMManagement = (function() {
|
||||
applyPolicyTemplate(button.dataset.createPolicyTemplate, createUserPoliciesEl);
|
||||
});
|
||||
});
|
||||
|
||||
var genAccessKeyBtn = document.getElementById('generateAccessKeyBtn');
|
||||
if (genAccessKeyBtn) {
|
||||
genAccessKeyBtn.addEventListener('click', function() {
|
||||
var input = document.getElementById('createUserAccessKey');
|
||||
if (input) input.value = generateSecureHex(8);
|
||||
});
|
||||
}
|
||||
|
||||
var genSecretKeyBtn = document.getElementById('generateSecretKeyBtn');
|
||||
if (genSecretKeyBtn) {
|
||||
genSecretKeyBtn.addEventListener('click', function() {
|
||||
var input = document.getElementById('createUserSecretKey');
|
||||
if (input) input.value = generateSecureBase64(24);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function setupEditUserModal() {
|
||||
@@ -242,23 +325,101 @@ window.IAMManagement = (function() {
|
||||
}
|
||||
}
|
||||
|
||||
function openExpiryModal(key, expiresAt) {
|
||||
currentExpiryKey = key;
|
||||
var label = document.getElementById('expiryUserLabel');
|
||||
var input = document.getElementById('expiryDateInput');
|
||||
var form = document.getElementById('expiryForm');
|
||||
if (label) label.textContent = key;
|
||||
if (expiresAt) {
|
||||
try {
|
||||
var dt = new Date(expiresAt);
|
||||
var local = new Date(dt.getTime() - dt.getTimezoneOffset() * 60000);
|
||||
if (input) input.value = local.toISOString().slice(0, 16);
|
||||
} catch(e) {
|
||||
if (input) input.value = '';
|
||||
}
|
||||
} else {
|
||||
if (input) input.value = '';
|
||||
}
|
||||
if (form) form.action = endpoints.updateExpiry.replace('ACCESS_KEY', key);
|
||||
var modalEl = document.getElementById('expiryModal');
|
||||
if (modalEl) {
|
||||
var modal = bootstrap.Modal.getOrCreateInstance(modalEl);
|
||||
modal.show();
|
||||
}
|
||||
}
|
||||
|
||||
function setupExpiryModal() {
|
||||
document.querySelectorAll('[data-expiry-user]').forEach(function(btn) {
|
||||
btn.addEventListener('click', function(e) {
|
||||
e.preventDefault();
|
||||
openExpiryModal(btn.dataset.expiryUser, btn.dataset.expiresAt || '');
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('[data-expiry-preset]').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
var preset = btn.dataset.expiryPreset;
|
||||
var input = document.getElementById('expiryDateInput');
|
||||
if (!input) return;
|
||||
if (preset === 'clear') {
|
||||
input.value = '';
|
||||
return;
|
||||
}
|
||||
var now = new Date();
|
||||
var ms = 0;
|
||||
if (preset === '1h') ms = 3600000;
|
||||
else if (preset === '24h') ms = 86400000;
|
||||
else if (preset === '7d') ms = 7 * 86400000;
|
||||
else if (preset === '30d') ms = 30 * 86400000;
|
||||
else if (preset === '90d') ms = 90 * 86400000;
|
||||
var future = new Date(now.getTime() + ms);
|
||||
var local = new Date(future.getTime() - future.getTimezoneOffset() * 60000);
|
||||
input.value = local.toISOString().slice(0, 16);
|
||||
});
|
||||
});
|
||||
|
||||
var expiryForm = document.getElementById('expiryForm');
|
||||
if (expiryForm) {
|
||||
expiryForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(expiryForm, {
|
||||
successMessage: 'Expiry updated',
|
||||
onSuccess: function() {
|
||||
var modalEl = document.getElementById('expiryModal');
|
||||
if (modalEl) bootstrap.Modal.getOrCreateInstance(modalEl).hide();
|
||||
window.location.reload();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function createUserCardHtml(accessKey, displayName, policies) {
|
||||
var admin = isAdminUser(policies);
|
||||
var cardClass = 'card h-100 iam-user-card' + (admin ? ' iam-admin-card' : '');
|
||||
var roleBadge = admin
|
||||
? '<span class="iam-role-badge iam-role-admin" data-role-badge>Admin</span>'
|
||||
: '<span class="iam-role-badge iam-role-user" data-role-badge>User</span>';
|
||||
|
||||
var policyBadges = '';
|
||||
if (policies && policies.length > 0) {
|
||||
policyBadges = policies.map(function(p) {
|
||||
var actionText = p.actions && p.actions.includes('*') ? 'full' : (p.actions ? p.actions.length : 0);
|
||||
return '<span class="badge bg-primary bg-opacity-10 text-primary">' +
|
||||
var bucketLabel = getBucketLabel(p.bucket);
|
||||
var permLevel = getPermissionLevel(p.actions);
|
||||
return '<span class="iam-perm-badge">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
|
||||
'<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>' +
|
||||
'</svg>' + window.UICore.escapeHtml(p.bucket) +
|
||||
'<span class="opacity-75">(' + actionText + ')</span></span>';
|
||||
'</svg>' + window.UICore.escapeHtml(bucketLabel) + ' · ' + window.UICore.escapeHtml(permLevel) + '</span>';
|
||||
}).join('');
|
||||
} else {
|
||||
policyBadges = '<span class="badge bg-secondary bg-opacity-10 text-secondary">No policies</span>';
|
||||
}
|
||||
|
||||
return '<div class="col-md-6 col-xl-4">' +
|
||||
'<div class="card h-100 iam-user-card">' +
|
||||
var esc = window.UICore.escapeHtml;
|
||||
return '<div class="col-md-6 col-xl-4 iam-user-item" data-display-name="' + esc(displayName.toLowerCase()) + '" data-access-key-filter="' + esc(accessKey.toLowerCase()) + '">' +
|
||||
'<div class="' + cardClass + '">' +
|
||||
'<div class="card-body">' +
|
||||
'<div class="d-flex align-items-start justify-content-between mb-3">' +
|
||||
'<div class="d-flex align-items-center gap-3 min-width-0 overflow-hidden">' +
|
||||
@@ -267,8 +428,18 @@ window.IAMManagement = (function() {
|
||||
'<path d="M8 8a3 3 0 1 0 0-6 3 3 0 0 0 0 6zm2-3a2 2 0 1 1-4 0 2 2 0 0 1 4 0zm4 8c0 1-1 1-1 1H3s-1 0-1-1 1-4 6-4 6 3 6 4zm-1-.004c-.001-.246-.154-.986-.832-1.664C11.516 10.68 10.289 10 8 10c-2.29 0-3.516.68-4.168 1.332-.678.678-.83 1.418-.832 1.664h10z"/>' +
|
||||
'</svg></div>' +
|
||||
'<div class="min-width-0">' +
|
||||
'<h6 class="fw-semibold mb-0 text-truncate" title="' + window.UICore.escapeHtml(displayName) + '">' + window.UICore.escapeHtml(displayName) + '</h6>' +
|
||||
'<code class="small text-muted d-block text-truncate" title="' + window.UICore.escapeHtml(accessKey) + '">' + window.UICore.escapeHtml(accessKey) + '</code>' +
|
||||
'<div class="d-flex align-items-center gap-2 mb-0">' +
|
||||
'<h6 class="fw-semibold mb-0 text-truncate" title="' + esc(displayName) + '">' + esc(displayName) + '</h6>' +
|
||||
roleBadge +
|
||||
'</div>' +
|
||||
'<div class="d-flex align-items-center gap-1">' +
|
||||
'<code class="small text-muted text-truncate" title="' + esc(accessKey) + '">' + esc(accessKey) + '</code>' +
|
||||
'<button type="button" class="iam-copy-key" title="Copy access key" data-copy-access-key="' + esc(accessKey) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">' +
|
||||
'<path d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>' +
|
||||
'<path d="M9.5 1a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1-.5-.5v-1a.5.5 0 0 1 .5-.5h3zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>' +
|
||||
'</svg></button>' +
|
||||
'</div>' +
|
||||
'</div></div>' +
|
||||
'<div class="dropdown flex-shrink-0">' +
|
||||
'<button class="btn btn-sm btn-icon" type="button" data-bs-toggle="dropdown" aria-expanded="false">' +
|
||||
@@ -276,18 +447,20 @@ window.IAMManagement = (function() {
|
||||
'<path d="M9.5 13a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0zm0-5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0zm0-5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0z"/>' +
|
||||
'</svg></button>' +
|
||||
'<ul class="dropdown-menu dropdown-menu-end">' +
|
||||
'<li><button class="dropdown-item" type="button" data-edit-user="' + window.UICore.escapeHtml(accessKey) + '" data-display-name="' + window.UICore.escapeHtml(displayName) + '">' +
|
||||
'<li><button class="dropdown-item" type="button" data-edit-user="' + esc(accessKey) + '" data-display-name="' + esc(displayName) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/></svg>Edit Name</button></li>' +
|
||||
'<li><button class="dropdown-item" type="button" data-rotate-user="' + window.UICore.escapeHtml(accessKey) + '">' +
|
||||
'<li><button class="dropdown-item" type="button" data-expiry-user="' + esc(accessKey) + '" data-expires-at="">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M8 3.5a.5.5 0 0 0-1 0V9a.5.5 0 0 0 .252.434l3.5 2a.5.5 0 0 0 .496-.868L8 8.71V3.5z"/><path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm7-8A7 7 0 1 1 1 8a7 7 0 0 1 14 0z"/></svg>Set Expiry</button></li>' +
|
||||
'<li><button class="dropdown-item" type="button" data-rotate-user="' + esc(accessKey) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M11.534 7h3.932a.25.25 0 0 1 .192.41l-1.966 2.36a.25.25 0 0 1-.384 0l-1.966-2.36a.25.25 0 0 1 .192-.41zm-11 2h3.932a.25.25 0 0 0 .192-.41L2.692 6.23a.25.25 0 0 0-.384 0L.342 8.59A.25.25 0 0 0 .534 9z"/><path fill-rule="evenodd" d="M8 3c-1.552 0-2.94.707-3.857 1.818a.5.5 0 1 1-.771-.636A6.002 6.002 0 0 1 13.917 7H12.9A5.002 5.002 0 0 0 8 3zM3.1 9a5.002 5.002 0 0 0 8.757 2.182.5.5 0 1 1 .771.636A6.002 6.002 0 0 1 2.083 9H3.1z"/></svg>Rotate Secret</button></li>' +
|
||||
'<li><hr class="dropdown-divider"></li>' +
|
||||
'<li><button class="dropdown-item text-danger" type="button" data-delete-user="' + window.UICore.escapeHtml(accessKey) + '">' +
|
||||
'<li><button class="dropdown-item text-danger" type="button" data-delete-user="' + esc(accessKey) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M5.5 5.5a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 .5-.5zm3 .5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 1 0z"/><path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/></svg>Delete User</button></li>' +
|
||||
'</ul></div></div>' +
|
||||
'<div class="mb-3">' +
|
||||
'<div class="small text-muted mb-2">Bucket Permissions</div>' +
|
||||
'<div class="d-flex flex-wrap gap-1">' + policyBadges + '</div></div>' +
|
||||
'<button class="btn btn-outline-primary btn-sm w-100" type="button" data-policy-editor data-access-key="' + window.UICore.escapeHtml(accessKey) + '">' +
|
||||
'<div class="d-flex flex-wrap gap-1" data-policy-badges>' + policyBadges + '</div></div>' +
|
||||
'<button class="btn btn-outline-primary btn-sm w-100" type="button" data-policy-editor data-access-key="' + esc(accessKey) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16"><path d="M8 4.754a3.246 3.246 0 1 0 0 6.492 3.246 3.246 0 0 0 0-6.492zM5.754 8a2.246 2.246 0 1 1 4.492 0 2.246 2.246 0 0 1-4.492 0z"/><path d="M9.796 1.343c-.527-1.79-3.065-1.79-3.592 0l-.094.319a.873.873 0 0 1-1.255.52l-.292-.16c-1.64-.892-3.433.902-2.54 2.541l.159.292a.873.873 0 0 1-.52 1.255l-.319.094c-1.79.527-1.79 3.065 0 3.592l.319.094a.873.873 0 0 1 .52 1.255l-.16.292c-.892 1.64.901 3.434 2.541 2.54l.292-.159a.873.873 0 0 1 1.255.52l.094.319c.527 1.79 3.065 1.79 3.592 0l.094-.319a.873.873 0 0 1 1.255-.52l.292.16c1.64.893 3.434-.902 2.54-2.541l-.159-.292a.873.873 0 0 1 .52-1.255l.319-.094c1.79-.527 1.79-3.065 0-3.592l-.319-.094a.873.873 0 0 1-.52-1.255l.16-.292c.893-1.64-.902-3.433-2.541-2.54l-.292.159a.873.873 0 0 1-1.255-.52l-.094-.319z"/></svg>Manage Policies</button>' +
|
||||
'</div></div></div>';
|
||||
}
|
||||
@@ -333,6 +506,14 @@ window.IAMManagement = (function() {
|
||||
});
|
||||
}
|
||||
|
||||
var expiryBtn = cardElement.querySelector('[data-expiry-user]');
|
||||
if (expiryBtn) {
|
||||
expiryBtn.addEventListener('click', function(e) {
|
||||
e.preventDefault();
|
||||
openExpiryModal(accessKey, '');
|
||||
});
|
||||
}
|
||||
|
||||
var policyBtn = cardElement.querySelector('[data-policy-editor]');
|
||||
if (policyBtn) {
|
||||
policyBtn.addEventListener('click', function() {
|
||||
@@ -342,6 +523,13 @@ window.IAMManagement = (function() {
|
||||
policyModal.show();
|
||||
});
|
||||
}
|
||||
|
||||
var copyBtn = cardElement.querySelector('[data-copy-access-key]');
|
||||
if (copyBtn) {
|
||||
copyBtn.addEventListener('click', function() {
|
||||
copyAccessKey(copyBtn);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function updateUserCount() {
|
||||
@@ -375,10 +563,15 @@ window.IAMManagement = (function() {
|
||||
'</svg>' +
|
||||
'<div class="flex-grow-1">' +
|
||||
'<div class="fw-semibold">New user created: <code>' + window.UICore.escapeHtml(data.access_key) + '</code></div>' +
|
||||
'<p class="mb-2 small">This secret is only shown once. Copy it now and store it securely.</p>' +
|
||||
'<p class="mb-2 small">These credentials are only shown once. Copy them now and store them securely.</p>' +
|
||||
'</div>' +
|
||||
'<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>' +
|
||||
'</div>' +
|
||||
'<div class="input-group mb-2">' +
|
||||
'<span class="input-group-text"><strong>Access key</strong></span>' +
|
||||
'<input class="form-control font-monospace" type="text" value="' + window.UICore.escapeHtml(data.access_key) + '" readonly />' +
|
||||
'<button class="btn btn-outline-primary" type="button" id="copyNewUserAccessKey">Copy</button>' +
|
||||
'</div>' +
|
||||
'<div class="input-group">' +
|
||||
'<span class="input-group-text"><strong>Secret key</strong></span>' +
|
||||
'<input class="form-control font-monospace" type="text" value="' + window.UICore.escapeHtml(data.secret_key) + '" readonly id="newUserSecret" />' +
|
||||
@@ -387,6 +580,9 @@ window.IAMManagement = (function() {
|
||||
var container = document.querySelector('.page-header');
|
||||
if (container) {
|
||||
container.insertAdjacentHTML('afterend', alertHtml);
|
||||
document.getElementById('copyNewUserAccessKey').addEventListener('click', async function() {
|
||||
await window.UICore.copyToClipboard(data.access_key, this, 'Copy');
|
||||
});
|
||||
document.getElementById('copyNewUserSecret').addEventListener('click', async function() {
|
||||
await window.UICore.copyToClipboard(data.secret_key, this, 'Copy');
|
||||
});
|
||||
@@ -442,17 +638,33 @@ window.IAMManagement = (function() {
|
||||
|
||||
var userCard = document.querySelector('[data-access-key="' + key + '"]');
|
||||
if (userCard) {
|
||||
var badgeContainer = userCard.closest('.iam-user-card').querySelector('.d-flex.flex-wrap.gap-1');
|
||||
var cardEl = userCard.closest('.iam-user-card');
|
||||
var badgeContainer = cardEl ? cardEl.querySelector('[data-policy-badges]') : null;
|
||||
if (badgeContainer && data.policies) {
|
||||
var badges = data.policies.map(function(p) {
|
||||
return '<span class="badge bg-primary bg-opacity-10 text-primary">' +
|
||||
var bl = getBucketLabel(p.bucket);
|
||||
var pl = getPermissionLevel(p.actions);
|
||||
return '<span class="iam-perm-badge">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
|
||||
'<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>' +
|
||||
'</svg>' + window.UICore.escapeHtml(p.bucket) +
|
||||
'<span class="opacity-75">(' + (p.actions.includes('*') ? 'full' : p.actions.length) + ')</span></span>';
|
||||
'</svg>' + window.UICore.escapeHtml(bl) + ' · ' + window.UICore.escapeHtml(pl) + '</span>';
|
||||
}).join('');
|
||||
badgeContainer.innerHTML = badges || '<span class="badge bg-secondary bg-opacity-10 text-secondary">No policies</span>';
|
||||
}
|
||||
if (cardEl) {
|
||||
var nowAdmin = isAdminUser(data.policies);
|
||||
cardEl.classList.toggle('iam-admin-card', nowAdmin);
|
||||
var roleBadgeEl = cardEl.querySelector('[data-role-badge]');
|
||||
if (roleBadgeEl) {
|
||||
if (nowAdmin) {
|
||||
roleBadgeEl.className = 'iam-role-badge iam-role-admin';
|
||||
roleBadgeEl.textContent = 'Admin';
|
||||
} else {
|
||||
roleBadgeEl.className = 'iam-role-badge iam-role-user';
|
||||
roleBadgeEl.textContent = 'User';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var userIndex = users.findIndex(function(u) { return u.access_key === key; });
|
||||
@@ -485,6 +697,10 @@ window.IAMManagement = (function() {
|
||||
nameEl.textContent = newName;
|
||||
nameEl.title = newName;
|
||||
}
|
||||
var itemWrapper = card.closest('.iam-user-item');
|
||||
if (itemWrapper) {
|
||||
itemWrapper.setAttribute('data-display-name', newName.toLowerCase());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -539,6 +755,52 @@ window.IAMManagement = (function() {
|
||||
}
|
||||
}
|
||||
|
||||
function setupSearch() {
|
||||
var searchInput = document.getElementById('iam-user-search');
|
||||
if (!searchInput) return;
|
||||
|
||||
searchInput.addEventListener('input', function() {
|
||||
var query = searchInput.value.toLowerCase().trim();
|
||||
var items = document.querySelectorAll('.iam-user-item');
|
||||
var noResults = document.getElementById('iam-no-results');
|
||||
var visibleCount = 0;
|
||||
|
||||
items.forEach(function(item) {
|
||||
var name = item.getAttribute('data-display-name') || '';
|
||||
var key = item.getAttribute('data-access-key-filter') || '';
|
||||
var matches = !query || name.indexOf(query) >= 0 || key.indexOf(query) >= 0;
|
||||
item.classList.toggle('d-none', !matches);
|
||||
if (matches) visibleCount++;
|
||||
});
|
||||
|
||||
if (noResults) {
|
||||
noResults.classList.toggle('d-none', visibleCount > 0);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function copyAccessKey(btn) {
|
||||
var key = btn.getAttribute('data-copy-access-key');
|
||||
if (!key) return;
|
||||
var originalHtml = btn.innerHTML;
|
||||
navigator.clipboard.writeText(key).then(function() {
|
||||
btn.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16"><path d="M13.854 3.646a.5.5 0 0 1 0 .708l-7 7a.5.5 0 0 1-.708 0l-3.5-3.5a.5.5 0 1 1 .708-.708L6.5 10.293l6.646-6.647a.5.5 0 0 1 .708 0z"/></svg>';
|
||||
btn.style.color = '#22c55e';
|
||||
setTimeout(function() {
|
||||
btn.innerHTML = originalHtml;
|
||||
btn.style.color = '';
|
||||
}, 1200);
|
||||
}).catch(function() {});
|
||||
}
|
||||
|
||||
function setupCopyAccessKeyButtons() {
|
||||
document.querySelectorAll('[data-copy-access-key]').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
copyAccessKey(btn);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
init: init
|
||||
};
|
||||
|
||||
@@ -191,6 +191,10 @@ window.UICore = (function() {
|
||||
}
|
||||
});
|
||||
|
||||
window.addEventListener('beforeunload', function() {
|
||||
pollingManager.stopAll();
|
||||
});
|
||||
|
||||
return {
|
||||
getCsrfToken: getCsrfToken,
|
||||
formatBytes: formatBytes,
|
||||
|
||||
@@ -101,6 +101,23 @@
|
||||
<span>Sites</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% if website_hosting_nav %}
|
||||
<a href="{{ url_for('ui.website_domains_dashboard') }}" class="sidebar-link {% if request.endpoint == 'ui.website_domains_dashboard' %}active{% endif %}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M4.715 6.542 3.343 7.914a3 3 0 1 0 4.243 4.243l1.828-1.829A3 3 0 0 0 8.586 5.5L8 6.086a1.002 1.002 0 0 0-.154.199 2 2 0 0 1 .861 3.337L6.88 11.45a2 2 0 1 1-2.83-2.83l.793-.792a4.018 4.018 0 0 1-.128-1.287z"/>
|
||||
<path d="M6.586 4.672A3 3 0 0 0 7.414 9.5l.775-.776a2 2 0 0 1-.896-3.346L9.12 3.55a2 2 0 1 1 2.83 2.83l-.793.792c.112.42.155.855.128 1.287l1.372-1.372a3 3 0 1 0-4.243-4.243L6.586 4.672z"/>
|
||||
</svg>
|
||||
<span>Domains</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% if can_manage_iam %}
|
||||
<a href="{{ url_for('ui.system_dashboard') }}" class="sidebar-link {% if request.endpoint == 'ui.system_dashboard' %}active{% endif %}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M9.405 1.05c-.413-1.4-2.397-1.4-2.81 0l-.1.34a1.464 1.464 0 0 1-2.105.872l-.31-.17c-1.283-.698-2.686.705-1.987 1.987l.169.311c.446.82.023 1.841-.872 2.105l-.34.1c-1.4.413-1.4 2.397 0 2.81l.34.1a1.464 1.464 0 0 1 .872 2.105l-.17.31c-.698 1.283.705 2.686 1.987 1.987l.311-.169a1.464 1.464 0 0 1 2.105.872l.1.34c.413 1.4 2.397 1.4 2.81 0l.1-.34a1.464 1.464 0 0 1 2.105-.872l.31.17c1.283.698 2.686-.705 1.987-1.987l-.169-.311a1.464 1.464 0 0 1 .872-2.105l.34-.1c1.4-.413 1.4-2.397 0-2.81l-.34-.1a1.464 1.464 0 0 1-.872-2.105l.17-.31c.698-1.283-.705-2.686-1.987-1.987l-.311.169a1.464 1.464 0 0 1-2.105-.872l-.1-.34zM8 10.93a2.929 2.929 0 1 1 0-5.86 2.929 2.929 0 0 1 0 5.858z"/>
|
||||
</svg>
|
||||
<span>System</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="nav-section">
|
||||
<span class="nav-section-title">Resources</span>
|
||||
@@ -192,6 +209,23 @@
|
||||
<span class="sidebar-link-text">Sites</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% if website_hosting_nav %}
|
||||
<a href="{{ url_for('ui.website_domains_dashboard') }}" class="sidebar-link {% if request.endpoint == 'ui.website_domains_dashboard' %}active{% endif %}" data-tooltip="Domains">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M4.715 6.542 3.343 7.914a3 3 0 1 0 4.243 4.243l1.828-1.829A3 3 0 0 0 8.586 5.5L8 6.086a1.002 1.002 0 0 0-.154.199 2 2 0 0 1 .861 3.337L6.88 11.45a2 2 0 1 1-2.83-2.83l.793-.792a4.018 4.018 0 0 1-.128-1.287z"/>
|
||||
<path d="M6.586 4.672A3 3 0 0 0 7.414 9.5l.775-.776a2 2 0 0 1-.896-3.346L9.12 3.55a2 2 0 1 1 2.83 2.83l-.793.792c.112.42.155.855.128 1.287l1.372-1.372a3 3 0 1 0-4.243-4.243L6.586 4.672z"/>
|
||||
</svg>
|
||||
<span class="sidebar-link-text">Domains</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% if can_manage_iam %}
|
||||
<a href="{{ url_for('ui.system_dashboard') }}" class="sidebar-link {% if request.endpoint == 'ui.system_dashboard' %}active{% endif %}" data-tooltip="System">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M9.405 1.05c-.413-1.4-2.397-1.4-2.81 0l-.1.34a1.464 1.464 0 0 1-2.105.872l-.31-.17c-1.283-.698-2.686.705-1.987 1.987l.169.311c.446.82.023 1.841-.872 2.105l-.34.1c-1.4.413-1.4 2.397 0 2.81l.34.1a1.464 1.464 0 0 1 .872 2.105l-.17.31c-.698 1.283.705 2.686 1.987 1.987l.311-.169a1.464 1.464 0 0 1 2.105.872l.1.34c.413 1.4 2.397 1.4 2.81 0l.1-.34a1.464 1.464 0 0 1 2.105-.872l.31.17c1.283.698 2.686-.705 1.987-1.987l-.169-.311a1.464 1.464 0 0 1 .872-2.105l.34-.1c1.4-.413 1.4-2.397 0-2.81l-.34-.1a1.464 1.464 0 0 1-.872-2.105l.17-.31c.698-1.283-.705-2.686-1.987-1.987l-.311.169a1.464 1.464 0 0 1-2.105-.872l-.1-.34zM8 10.93a2.929 2.929 0 1 1 0-5.86 2.929 2.929 0 0 1 0 5.858z"/>
|
||||
</svg>
|
||||
<span class="sidebar-link-text">System</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="nav-section">
|
||||
<span class="nav-section-title">Resources</span>
|
||||
|
||||
@@ -100,8 +100,26 @@
|
||||
</svg>
|
||||
Upload
|
||||
</button>
|
||||
<div class="dropdown sort-dropdown">
|
||||
<button class="btn btn-outline-secondary btn-sm dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" title="Sort objects">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M3.5 2.5a.5.5 0 0 0-1 0v8.793l-1.146-1.147a.5.5 0 0 0-.708.708l2 1.999.007.007a.497.497 0 0 0 .7-.006l2-2a.5.5 0 0 0-.707-.708L3.5 11.293V2.5zm3.5 1a.5.5 0 0 1 .5-.5h7a.5.5 0 0 1 0 1h-7a.5.5 0 0 1-.5-.5zM7.5 6a.5.5 0 0 0 0 1h5a.5.5 0 0 0 0-1h-5zm0 3a.5.5 0 0 0 0 1h3a.5.5 0 0 0 0-1h-3zm0 3a.5.5 0 0 0 0 1h1a.5.5 0 0 0 0-1h-1z"/>
|
||||
</svg>
|
||||
<span id="sort-dropdown-label">Name A-Z</span>
|
||||
</button>
|
||||
<ul class="dropdown-menu dropdown-menu-end">
|
||||
<li><button class="dropdown-item active" type="button" data-sort-field="name" data-sort-dir="asc">Name A-Z</button></li>
|
||||
<li><button class="dropdown-item" type="button" data-sort-field="name" data-sort-dir="desc">Name Z-A</button></li>
|
||||
<li><hr class="dropdown-divider"></li>
|
||||
<li><button class="dropdown-item" type="button" data-sort-field="size" data-sort-dir="desc">Size (largest)</button></li>
|
||||
<li><button class="dropdown-item" type="button" data-sort-field="size" data-sort-dir="asc">Size (smallest)</button></li>
|
||||
<li><hr class="dropdown-divider"></li>
|
||||
<li><button class="dropdown-item" type="button" data-sort-field="date" data-sort-dir="desc">Date (newest)</button></li>
|
||||
<li><button class="dropdown-item" type="button" data-sort-field="date" data-sort-dir="asc">Date (oldest)</button></li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="position-relative search-wrapper">
|
||||
<input id="object-search" class="form-control form-control-sm" type="search" placeholder="Filter objects" style="max-width: 180px;" />
|
||||
<input id="object-search" class="form-control form-control-sm" type="search" placeholder="Filter objects (press /)" style="max-width: 180px;" />
|
||||
</div>
|
||||
<div class="bulk-actions d-none" id="bulk-actions-wrapper">
|
||||
<button class="btn btn-outline-danger btn-sm" type="button" data-bulk-delete-trigger disabled>
|
||||
@@ -153,6 +171,7 @@
|
||||
data-bulk-download-endpoint="{{ url_for('ui.bulk_download_objects', bucket_name=bucket_name) }}"
|
||||
data-folders-url="{{ folders_url }}"
|
||||
data-buckets-for-copy-url="{{ buckets_for_copy_url }}"
|
||||
data-bucket-total-objects="{{ bucket_stats.get('objects', 0) }}"
|
||||
>
|
||||
<table class="table table-hover align-middle mb-0" id="objects-table" style="table-layout: fixed;">
|
||||
<thead class="table-light">
|
||||
@@ -238,7 +257,8 @@
|
||||
Share Link
|
||||
</button>
|
||||
</div>
|
||||
<div class="p-3 rounded mb-3" style="background: var(--myfsio-preview-bg);">
|
||||
<div id="preview-error-alert" class="alert alert-warning d-none py-2 px-3 mb-3 small" role="alert"></div>
|
||||
<div id="preview-details-meta" class="p-3 rounded mb-3" style="background: var(--myfsio-preview-bg);">
|
||||
<dl class="row small mb-0">
|
||||
<dt class="col-5 text-muted fw-normal">Last modified</dt>
|
||||
<dd class="col-7 mb-2 fw-medium" id="preview-modified"></dd>
|
||||
@@ -273,19 +293,28 @@
|
||||
Edit
|
||||
</button>
|
||||
</div>
|
||||
<div id="preview-tags-list" class="d-flex flex-wrap gap-1"></div>
|
||||
<div id="preview-tags-list" class="d-flex flex-wrap gap-2"></div>
|
||||
<div id="preview-tags-empty" class="text-muted small p-2 bg-body-tertiary rounded">No tags</div>
|
||||
<div id="preview-tags-editor" class="d-none mt-2">
|
||||
<div id="preview-tags-inputs" class="mb-2"></div>
|
||||
<div class="d-flex gap-2">
|
||||
<button class="btn btn-sm btn-outline-secondary flex-grow-1" type="button" id="addTagRow">
|
||||
<div class="tag-editor-card">
|
||||
<div class="tag-editor-header">
|
||||
<span>Key</span>
|
||||
<span>Value</span>
|
||||
<span></span>
|
||||
</div>
|
||||
<div id="preview-tags-inputs"></div>
|
||||
<div class="tag-editor-actions">
|
||||
<button class="btn btn-sm btn-outline-secondary" type="button" id="addTagRow">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M8 4a.5.5 0 0 1 .5.5v3h3a.5.5 0 0 1 0 1h-3v3a.5.5 0 0 1-1 0v-3h-3a.5.5 0 0 1 0-1h3v-3A.5.5 0 0 1 8 4z"/>
|
||||
</svg>
|
||||
Add Tag
|
||||
</button>
|
||||
<button class="btn btn-sm btn-primary" type="button" id="saveTagsButton">Save</button>
|
||||
<div class="ms-auto d-flex gap-2">
|
||||
<button class="btn btn-sm btn-outline-secondary" type="button" id="cancelTagsButton">Cancel</button>
|
||||
<button class="btn btn-sm btn-primary" type="button" id="saveTagsButton">Save</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-text mt-1">Maximum 10 tags. Keys and values up to 256 characters.</div>
|
||||
</div>
|
||||
@@ -321,7 +350,8 @@
|
||||
<img id="preview-image" class="img-fluid d-none w-100" alt="Object preview" style="display: block;" />
|
||||
<video id="preview-video" class="w-100 d-none" controls style="display: block;"></video>
|
||||
<audio id="preview-audio" class="w-100 d-none" controls style="display: block;"></audio>
|
||||
<iframe id="preview-iframe" class="w-100 d-none" loading="lazy" style="min-height: 200px;"></iframe>
|
||||
<pre id="preview-text" class="w-100 d-none m-0"></pre>
|
||||
<iframe id="preview-iframe" class="w-100 d-none" style="min-height: 200px;"></iframe>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -965,6 +995,111 @@
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if website_hosting_enabled %}
|
||||
<div class="card shadow-sm mt-4" id="bucket-website-card">
|
||||
<div class="card-header d-flex align-items-center">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" class="text-primary me-2" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
<span class="fw-semibold">Static Website Hosting</span>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
{% if website_config %}
|
||||
<div class="alert alert-success d-flex align-items-start mb-4" role="alert">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="me-2 flex-shrink-0" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>
|
||||
</svg>
|
||||
<div>
|
||||
<strong>Website hosting is enabled</strong>
|
||||
<p class="mb-0 small">
|
||||
Index: <code>{{ website_config.index_document }}</code>
|
||||
{% if website_config.error_document %}<br>Error: <code>{{ website_config.error_document }}</code>{% endif %}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="alert alert-secondary d-flex align-items-start mb-4" role="alert">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="me-2 flex-shrink-0" viewBox="0 0 16 16">
|
||||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||||
<path d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
|
||||
</svg>
|
||||
<div>
|
||||
<strong>Website hosting is disabled</strong>
|
||||
<p class="mb-0 small">Enable website hosting to serve bucket contents as a static website.</p>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if website_domains %}
|
||||
<div class="mb-4">
|
||||
<label class="form-label fw-medium mb-2">Mapped Domains</label>
|
||||
{% for domain in website_domains %}
|
||||
<div class="d-flex align-items-center mb-1">
|
||||
<span class="badge bg-success-subtle text-success-emphasis me-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M7.21 1.293a1 1 0 0 1 1.58 0l.612.72a1 1 0 0 0 .88.332l.94-.134a1 1 0 0 1 1.118.7l.248.912a1 1 0 0 0 .59.659l.876.388a1 1 0 0 1 .435 1.505l-.546.766a1 1 0 0 0-.156.935l.306.899a1 1 0 0 1-.725 1.282l-.92.216a1 1 0 0 0-.72.555l-.41.856a1 1 0 0 1-1.396.478l-.803-.49a1 1 0 0 0-1.04 0l-.802.49a1 1 0 0 1-1.397-.478l-.41-.857a1 1 0 0 0-.72-.554l-.919-.216a1 1 0 0 1-.725-1.282l.306-.9a1 1 0 0 0-.156-.934l-.546-.766a1 1 0 0 1 .435-1.505l.877-.388a1 1 0 0 0 .589-.66l.248-.911a1 1 0 0 1 1.118-.7l.94.133a1 1 0 0 0 .88-.331l.612-.72zM11 7a.5.5 0 0 0-.5-.5h-5a.5.5 0 0 0 0 1H6v1.5a.5.5 0 0 0 1 0V7.5h1v2a.5.5 0 0 0 1 0v-2h1.5a.5.5 0 0 0 0-1H10V7z"/>
|
||||
</svg>
|
||||
connected
|
||||
</span>
|
||||
<code class="small">{{ domain }}</code>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% elif website_config %}
|
||||
<div class="mb-4">
|
||||
<label class="form-label fw-medium mb-2">Mapped Domains</label>
|
||||
<p class="text-muted small mb-0">No domains mapped to this bucket. <a href="{{ url_for('ui.website_domains_dashboard') }}">Manage domains</a></p>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if can_manage_website %}
|
||||
<form method="post" action="{{ url_for('ui.update_bucket_website', bucket_name=bucket_name) }}" id="websiteForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
|
||||
<input type="hidden" name="action" value="enable" id="websiteAction" />
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="index_document" class="form-label fw-medium">Index Document</label>
|
||||
<input type="text" class="form-control" id="index_document" name="index_document"
|
||||
value="{{ website_config.index_document if website_config else 'index.html' }}"
|
||||
placeholder="index.html">
|
||||
<div class="form-text">The default page served for directory paths (e.g., index.html).</div>
|
||||
</div>
|
||||
|
||||
<div class="mb-4">
|
||||
<label for="error_document" class="form-label fw-medium">Error Document</label>
|
||||
<input type="text" class="form-control" id="error_document" name="error_document"
|
||||
value="{{ website_config.error_document if website_config else '' }}"
|
||||
placeholder="error.html">
|
||||
<div class="form-text">Optional. The page served for 404 errors.</div>
|
||||
</div>
|
||||
|
||||
<div class="d-flex gap-2 flex-wrap">
|
||||
<button class="btn {{ 'btn-primary' if website_config else 'btn-success' }}" type="submit" id="websiteSubmitBtn">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M12.736 3.97a.733.733 0 0 1 1.047 0c.286.289.29.756.01 1.05L7.88 12.01a.733.733 0 0 1-1.065.02L3.217 8.384a.757.757 0 0 1 0-1.06.733.733 0 0 1 1.047 0l3.052 3.093 5.4-6.425a.247.247 0 0 1 .02-.022Z"/>
|
||||
</svg>
|
||||
<span id="websiteSubmitLabel">{{ 'Save Website Settings' if website_config else 'Enable Website Hosting' }}</span>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-danger" id="disableWebsiteBtn"{% if not website_config %} style="display: none;"{% endif %}>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
|
||||
</svg>
|
||||
Disable Website Hosting
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
{% else %}
|
||||
<div class="text-center py-3">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="currentColor" class="text-muted mb-2" viewBox="0 0 16 16">
|
||||
<path d="M8 1a2 2 0 0 1 2 2v4H6V3a2 2 0 0 1 2-2zm3 6V3a3 3 0 0 0-6 0v4a2 2 0 0 0-2 2v5a2 2 0 0 0 2 2h6a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2z"/>
|
||||
</svg>
|
||||
<p class="text-muted mb-0 small">You do not have permission to modify website hosting for this bucket.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="col-lg-4">
|
||||
@@ -1923,7 +2058,7 @@
|
||||
<div class="col-12">
|
||||
<label class="form-label fw-medium">Select files</label>
|
||||
<input class="form-control" type="file" name="object" id="uploadFileInput" multiple required />
|
||||
<div class="form-text">Select one or more files from your device. Files ≥ 8 MB automatically switch to multipart uploads.</div>
|
||||
<div class="form-text">Select one or more files from your device. Files ≥ 8 MB use multipart uploads with automatic retry.</div>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<div class="upload-dropzone text-center" data-dropzone>
|
||||
@@ -2148,13 +2283,11 @@
|
||||
</div>
|
||||
<ul class="list-group mb-3" id="bulkDeleteList" style="max-height: 200px; overflow-y: auto;"></ul>
|
||||
<div class="text-muted small" id="bulkDeleteStatus"></div>
|
||||
{% if versioning_enabled %}
|
||||
<div class="form-check mt-3 p-3 bg-body-tertiary rounded-3">
|
||||
<div class="form-check mt-3 p-3 bg-body-tertiary rounded-3 {% if not versioning_enabled %}d-none{% endif %}" id="bulkDeletePurgeWrap">
|
||||
<input class="form-check-input" type="checkbox" id="bulkDeletePurge" />
|
||||
<label class="form-check-label" for="bulkDeletePurge">Also delete archived versions</label>
|
||||
<div class="form-text">Removes any archived versions stored in the archive.</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
@@ -2192,7 +2325,7 @@
|
||||
<div class="p-3 bg-body-tertiary rounded-3 mb-3">
|
||||
<code id="deleteObjectKey" class="d-block text-break"></code>
|
||||
</div>
|
||||
{% if versioning_enabled %}
|
||||
<div id="deleteObjectVersioningWrap" class="{% if not versioning_enabled %}d-none{% endif %}">
|
||||
<div class="alert alert-warning d-flex align-items-start small mb-3" role="alert">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="flex-shrink-0 me-2 mt-0" viewBox="0 0 16 16">
|
||||
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
|
||||
@@ -2204,7 +2337,7 @@
|
||||
<label class="form-check-label" for="deletePurgeVersions">Also delete all archived versions</label>
|
||||
<div class="form-text mb-0">Removes the live object and every stored version.</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
@@ -2579,6 +2712,63 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="context-menu d-none" id="objectContextMenu">
|
||||
<button class="context-menu-item" data-ctx-action="download">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M.5 9.9a.5.5 0 0 1 .5.5v2.5a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1v-2.5a.5.5 0 0 1 1 0v2.5a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2v-2.5a.5.5 0 0 1 .5-.5z"/>
|
||||
<path d="M7.646 11.854a.5.5 0 0 0 .708 0l3-3a.5.5 0 0 0-.708-.708L8.5 10.293V1.5a.5.5 0 0 0-1 0v8.793L5.354 8.146a.5.5 0 1 0-.708.708l3 3z"/>
|
||||
</svg>
|
||||
Download
|
||||
</button>
|
||||
<button class="context-menu-item" data-ctx-action="copy-path">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M4 2a2 2 0 0 1 2-2h8a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V2Zm2-1a1 1 0 0 0-1 1v8a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1H6ZM2 5a1 1 0 0 0-1 1v8a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1v-1h1v1a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h1v1H2Z"/>
|
||||
</svg>
|
||||
Copy S3 Path
|
||||
</button>
|
||||
<button class="context-menu-item" data-ctx-action="presign">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M4.715 6.542 3.343 7.914a3 3 0 1 0 4.243 4.243l1.828-1.829A3 3 0 0 0 8.586 5.5L8 6.086a1.002 1.002 0 0 0-.154.199 2 2 0 0 1 .861 3.337L6.88 11.45a2 2 0 1 1-2.83-2.83l.793-.792a4.018 4.018 0 0 1-.128-1.287z"/>
|
||||
<path d="M6.586 4.672A3 3 0 0 0 7.414 9.5l.775-.776a2 2 0 0 1-.896-3.346L9.12 3.55a2 2 0 1 1 2.83 2.83l-.793.792c.112.42.155.855.128 1.287l1.372-1.372a3 3 0 1 0-4.243-4.243L6.586 4.672z"/>
|
||||
</svg>
|
||||
Share Link
|
||||
</button>
|
||||
<div class="context-menu-divider"></div>
|
||||
<button class="context-menu-item text-danger" data-ctx-action="delete">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="keyboardShortcutsModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered modal-sm">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header border-0 pb-0">
|
||||
<h5 class="modal-title fw-semibold">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" class="text-primary me-1" viewBox="0 0 16 16">
|
||||
<path d="M14 5a1 1 0 0 1 1 1v5a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V6a1 1 0 0 1 1-1h12zM2 4a2 2 0 0 0-2 2v5a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2H2z"/>
|
||||
<path d="M13 10.25a.25.25 0 0 1 .25-.25h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5a.25.25 0 0 1-.25-.25v-.5zm0-2a.25.25 0 0 1 .25-.25h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5a.25.25 0 0 1-.25-.25v-.5zm-5 0A.25.25 0 0 1 8.25 8h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5A.25.25 0 0 1 8 8.75v-.5zm2 0a.25.25 0 0 1 .25-.25h1.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-1.5a.25.25 0 0 1-.25-.25v-.5zm1 2a.25.25 0 0 1 .25-.25h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5a.25.25 0 0 1-.25-.25v-.5zm-5-2A.25.25 0 0 1 6.25 8h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5A.25.25 0 0 1 6 8.75v-.5zm-2 0A.25.25 0 0 1 4.25 8h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5A.25.25 0 0 1 4 8.75v-.5zm-2 0A.25.25 0 0 1 2.25 8h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5A.25.25 0 0 1 2 8.75v-.5zm11-2a.25.25 0 0 1 .25-.25h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5a.25.25 0 0 1-.25-.25v-.5zm-2 0a.25.25 0 0 1 .25-.25h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5a.25.25 0 0 1-.25-.25v-.5zm-2 0A.25.25 0 0 1 9.25 6h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5A.25.25 0 0 1 9 6.75v-.5zm-2 0A.25.25 0 0 1 7.25 6h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5A.25.25 0 0 1 7 6.75v-.5zm-2 0A.25.25 0 0 1 5.25 6h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5A.25.25 0 0 1 5 6.75v-.5zm-3 0A.25.25 0 0 1 2.25 6h1.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-1.5A.25.25 0 0 1 2 6.75v-.5zm0 4a.25.25 0 0 1 .25-.25h.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-.5a.25.25 0 0 1-.25-.25v-.5zm2 0a.25.25 0 0 1 .25-.25h5.5a.25.25 0 0 1 .25.25v.5a.25.25 0 0 1-.25.25h-5.5a.25.25 0 0 1-.25-.25v-.5z"/>
|
||||
</svg>
|
||||
Keyboard Shortcuts
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body pt-2">
|
||||
<div class="kbd-shortcuts-list">
|
||||
<div class="shortcut-row"><span class="text-muted">Search objects</span><kbd>/</kbd></div>
|
||||
<div class="shortcut-row"><span class="text-muted">Select all</span><span><kbd>Ctrl</kbd> + <kbd>A</kbd></span></div>
|
||||
<div class="shortcut-row"><span class="text-muted">Delete selected</span><kbd>Del</kbd></div>
|
||||
<div class="shortcut-row"><span class="text-muted">Clear search</span><kbd>Esc</kbd></div>
|
||||
<div class="shortcut-row"><span class="text-muted">Show shortcuts</span><kbd>?</kbd></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block extra_scripts %}
|
||||
@@ -2590,7 +2780,8 @@
|
||||
window.BucketDetailConfig = {
|
||||
endpoints: {
|
||||
versioning: "{{ url_for('ui.update_bucket_versioning', bucket_name=bucket_name) }}",
|
||||
bucketsOverview: "{{ url_for('ui.buckets_overview') }}"
|
||||
bucketsOverview: "{{ url_for('ui.buckets_overview') }}",
|
||||
archivedObjects: "{{ url_for('ui.archived_objects', bucket_name=bucket_name) }}"
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@
|
||||
</div>
|
||||
<div>
|
||||
<h5 class="bucket-name text-break">{{ bucket.meta.name }}</h5>
|
||||
<small class="text-muted">Created {{ bucket.meta.created_at | format_datetime }}</small>
|
||||
<small class="text-muted">Created {{ bucket.meta.creation_date | format_datetime }}</small>
|
||||
</div>
|
||||
</div>
|
||||
<span class="badge {{ bucket.access_badge }} bucket-access-badge">{{ bucket.access_label }}</span>
|
||||
@@ -89,6 +89,14 @@
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<div class="col-12 d-none" id="bucket-no-results">
|
||||
<div class="text-center py-5 text-muted">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="currentColor" class="mb-3 opacity-50" viewBox="0 0 16 16">
|
||||
<path d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0z"/>
|
||||
</svg>
|
||||
<p class="mb-0 fw-medium">No buckets match your filter.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="createBucketModal" tabindex="-1" aria-hidden="true">
|
||||
@@ -141,7 +149,7 @@
|
||||
let visibleCount = 0;
|
||||
|
||||
bucketItems.forEach(item => {
|
||||
const name = item.querySelector('.card-title').textContent.toLowerCase();
|
||||
const name = item.querySelector('.bucket-name').textContent.toLowerCase();
|
||||
if (name.includes(term)) {
|
||||
item.classList.remove('d-none');
|
||||
visibleCount++;
|
||||
@@ -149,6 +157,15 @@
|
||||
item.classList.add('d-none');
|
||||
}
|
||||
});
|
||||
|
||||
var noResults = document.getElementById('bucket-no-results');
|
||||
if (noResults) {
|
||||
if (term && visibleCount === 0) {
|
||||
noResults.classList.remove('d-none');
|
||||
} else {
|
||||
noResults.classList.add('d-none');
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -145,7 +145,6 @@
|
||||
data-endpoint="{{ conn.endpoint_url }}"
|
||||
data-region="{{ conn.region }}"
|
||||
data-access="{{ conn.access_key }}"
|
||||
data-secret="{{ conn.secret_key }}"
|
||||
title="Edit connection">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
|
||||
|
||||
@@ -40,6 +40,8 @@
|
||||
<li><a href="#quotas">Bucket Quotas</a></li>
|
||||
<li><a href="#encryption">Encryption</a></li>
|
||||
<li><a href="#lifecycle">Lifecycle Rules</a></li>
|
||||
<li><a href="#garbage-collection">Garbage Collection</a></li>
|
||||
<li><a href="#integrity">Integrity Scanner</a></li>
|
||||
<li><a href="#metrics">Metrics History</a></li>
|
||||
<li><a href="#operation-metrics">Operation Metrics</a></li>
|
||||
<li><a href="#troubleshooting">Troubleshooting</a></li>
|
||||
@@ -51,6 +53,12 @@
|
||||
<li><a href="#advanced-ops">Advanced Operations</a></li>
|
||||
<li><a href="#acls">Access Control Lists</a></li>
|
||||
<li><a href="#tagging">Object & Bucket Tagging</a></li>
|
||||
<li><a href="#website-hosting">Static Website Hosting</a></li>
|
||||
<li><a href="#cors-config">CORS Configuration</a></li>
|
||||
<li><a href="#post-object">PostObject (Form Upload)</a></li>
|
||||
<li><a href="#list-objects-v2">List Objects API v2</a></li>
|
||||
<li><a href="#upgrading">Upgrading & Updates</a></li>
|
||||
<li><a href="#api-matrix">Full API Reference</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
@@ -76,7 +84,7 @@ pip install -r requirements.txt
|
||||
# Run both API and UI (Development)
|
||||
python run.py
|
||||
|
||||
# Run in Production (Waitress server)
|
||||
# Run in Production (Granian server)
|
||||
python run.py --prod
|
||||
|
||||
# Or run individually
|
||||
@@ -97,8 +105,8 @@ python run.py --mode ui
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>API_BASE_URL</code></td>
|
||||
<td><code>None</code></td>
|
||||
<td>The public URL of the API. <strong>Required</strong> if running behind a proxy. Ensures presigned URLs are generated correctly.</td>
|
||||
<td><code>http://127.0.0.1:5000</code></td>
|
||||
<td>Internal S3 API URL used by the web UI proxy. Also used for presigned URL generation. Set to your public URL if running behind a reverse proxy.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>STORAGE_ROOT</code></td>
|
||||
@@ -125,6 +133,11 @@ python run.py --mode ui
|
||||
<td><code>5000</code></td>
|
||||
<td>Listen port (UI uses 5100).</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>DISPLAY_TIMEZONE</code></td>
|
||||
<td><code>UTC</code></td>
|
||||
<td>Timezone for UI timestamps (e.g., <code>US/Eastern</code>, <code>Asia/Tokyo</code>).</td>
|
||||
</tr>
|
||||
<tr class="table-secondary">
|
||||
<td colspan="3" class="fw-semibold">CORS Settings</td>
|
||||
</tr>
|
||||
@@ -186,13 +199,28 @@ python run.py --mode ui
|
||||
<td><code>100 per minute</code></td>
|
||||
<td>Rate limit for HEAD requests.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>RATE_LIMIT_ADMIN</code></td>
|
||||
<td><code>60 per minute</code></td>
|
||||
<td>Rate limit for admin API endpoints (<code>/admin/*</code>).</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>ADMIN_ACCESS_KEY</code></td>
|
||||
<td>(none)</td>
|
||||
<td>Custom access key for the admin user on first run or credential reset. Random if unset.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>ADMIN_SECRET_KEY</code></td>
|
||||
<td>(none)</td>
|
||||
<td>Custom secret key for the admin user on first run or credential reset. Random if unset.</td>
|
||||
</tr>
|
||||
<tr class="table-secondary">
|
||||
<td colspan="3" class="fw-semibold">Server Settings</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>SERVER_THREADS</code></td>
|
||||
<td><code>0</code> (auto)</td>
|
||||
<td>Waitress worker threads (1-64). 0 = auto (CPU cores × 2).</td>
|
||||
<td>Granian blocking threads (1-64). 0 = auto (CPU cores × 2).</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>SERVER_CONNECTION_LIMIT</code></td>
|
||||
@@ -337,6 +365,24 @@ python run.py --mode ui
|
||||
<td><code>604800</code></td>
|
||||
<td>Maximum presigned URL expiry time (7 days).</td>
|
||||
</tr>
|
||||
<tr class="table-secondary">
|
||||
<td colspan="3" class="fw-semibold">Proxy & Network Settings</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>NUM_TRUSTED_PROXIES</code></td>
|
||||
<td><code>1</code></td>
|
||||
<td>Number of trusted reverse proxies for <code>X-Forwarded-*</code> headers.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>ALLOWED_REDIRECT_HOSTS</code></td>
|
||||
<td>(empty)</td>
|
||||
<td>Comma-separated whitelist of safe redirect targets.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>ALLOW_INTERNAL_ENDPOINTS</code></td>
|
||||
<td><code>false</code></td>
|
||||
<td>Allow connections to internal/private IPs (webhooks, replication).</td>
|
||||
</tr>
|
||||
<tr class="table-secondary">
|
||||
<td colspan="3" class="fw-semibold">Storage Limits</td>
|
||||
</tr>
|
||||
@@ -365,6 +411,16 @@ python run.py --mode ui
|
||||
<td><code>50</code></td>
|
||||
<td>Max lifecycle history records per bucket.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>OBJECT_CACHE_TTL</code></td>
|
||||
<td><code>60</code></td>
|
||||
<td>Seconds to cache object metadata.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>BULK_DOWNLOAD_MAX_BYTES</code></td>
|
||||
<td><code>1 GB</code></td>
|
||||
<td>Max total size for bulk ZIP downloads.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>ENCRYPTION_CHUNK_SIZE_BYTES</code></td>
|
||||
<td><code>65536</code></td>
|
||||
@@ -384,7 +440,7 @@ python run.py --mode ui
|
||||
</table>
|
||||
</div>
|
||||
<div class="alert alert-warning mt-3 mb-0 small">
|
||||
<strong>Production Checklist:</strong> Set <code>SECRET_KEY</code>, restrict <code>CORS_ORIGINS</code>, configure <code>API_BASE_URL</code>, enable HTTPS via reverse proxy, and use <code>--prod</code> flag.
|
||||
<strong>Production Checklist:</strong> Set <code>SECRET_KEY</code> (also enables IAM config encryption at rest), restrict <code>CORS_ORIGINS</code>, configure <code>API_BASE_URL</code>, enable HTTPS via reverse proxy, use <code>--prod</code> flag, and set credential expiry on non-admin users.
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
@@ -451,11 +507,12 @@ sudo journalctl -u myfsio -f # View logs</code></pre>
|
||||
<span class="docs-section-kicker">03</span>
|
||||
<h2 class="h4 mb-0">Authenticate & manage IAM</h2>
|
||||
</div>
|
||||
<p class="text-muted">On first startup, MyFSIO generates random admin credentials and prints them to the console. Missed it? Check <code>data/.myfsio.sys/config/iam.json</code> directly—credentials are stored in plaintext.</p>
|
||||
<p class="text-muted">On first startup, MyFSIO generates random admin credentials and prints them to the console. Set <code>ADMIN_ACCESS_KEY</code> and <code>ADMIN_SECRET_KEY</code> env vars for custom credentials. When <code>SECRET_KEY</code> is configured, the IAM config is encrypted at rest. To reset credentials, run <code>python run.py --reset-cred</code>.</p>
|
||||
<div class="docs-highlight mb-3">
|
||||
<ol class="mb-0">
|
||||
<li>Check the console output (or <code>iam.json</code>) for the generated <code>Access Key</code> and <code>Secret Key</code>, then visit <code>/ui/login</code>.</li>
|
||||
<li>Create additional users with descriptive display names and AWS-style inline policies (for example <code>{"bucket": "*", "actions": ["list", "read"]}</code>).</li>
|
||||
<li>Check the console output for the generated <code>Access Key</code> and <code>Secret Key</code>, then visit <code>/ui/login</code>.</li>
|
||||
<li>Create additional users with descriptive display names, AWS-style inline policies (for example <code>{"bucket": "*", "actions": ["list", "read"]}</code>), and optional credential expiry dates.</li>
|
||||
<li>Set credential expiry on users to grant time-limited access. The UI shows expiry badges and provides preset durations (1h, 24h, 7d, 30d, 90d). Expired credentials are rejected at authentication.</li>
|
||||
<li>Rotate secrets when sharing with CI jobs—new secrets display once and persist to <code>data/.myfsio.sys/config/iam.json</code>.</li>
|
||||
<li>Bucket policies layer on top of IAM. Apply Private/Public presets or paste custom JSON; changes reload instantly.</li>
|
||||
</ol>
|
||||
@@ -490,7 +547,7 @@ sudo journalctl -u myfsio -f # View logs</code></pre>
|
||||
<ul>
|
||||
<li>Navigate folder hierarchies using breadcrumbs. Objects with <code>/</code> in keys display as folders.</li>
|
||||
<li>Infinite scroll loads more objects automatically. Choose batch size (50–250) from the footer dropdown.</li>
|
||||
<li>Bulk select objects for multi-delete or multi-download. Filter by name using the search box.</li>
|
||||
<li>Bulk select objects for multi-delete or multi-download (ZIP archive, up to 1 GiB). Filter by name using the search box.</li>
|
||||
<li>If loading fails, click <strong>Retry</strong> to attempt again—no page refresh needed.</li>
|
||||
</ul>
|
||||
</div>
|
||||
@@ -612,15 +669,75 @@ curl -X PUT {{ api_base }}/demo/notes.txt \
|
||||
<td><code>/<bucket>/<key></code></td>
|
||||
<td>Delete an object.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>HEAD</td>
|
||||
<td><code>/<bucket></code></td>
|
||||
<td>Check if a bucket exists.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>HEAD</td>
|
||||
<td><code>/<bucket>/<key></code></td>
|
||||
<td>Get object metadata without downloading.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>POST</td>
|
||||
<td><code>/<bucket>?delete</code></td>
|
||||
<td>Bulk delete objects (XML body).</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GET/PUT/DELETE</td>
|
||||
<td><code>/<bucket>?policy</code></td>
|
||||
<td>Fetch, upsert, or remove a bucket policy (S3-compatible).</td>
|
||||
<td>Bucket policy management.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GET/PUT</td>
|
||||
<td><code>/<bucket>?versioning</code></td>
|
||||
<td>Versioning status.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GET/PUT/DELETE</td>
|
||||
<td><code>/<bucket>?lifecycle</code></td>
|
||||
<td>Lifecycle rules.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GET/PUT/DELETE</td>
|
||||
<td><code>/<bucket>?cors</code></td>
|
||||
<td>CORS configuration.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GET/PUT/DELETE</td>
|
||||
<td><code>/<bucket>?encryption</code></td>
|
||||
<td>Default encryption.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GET/PUT</td>
|
||||
<td><code>/<bucket>?acl</code></td>
|
||||
<td>Bucket ACL.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GET/PUT/DELETE</td>
|
||||
<td><code>/<bucket>?tagging</code></td>
|
||||
<td>Bucket tags.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>GET/PUT/DELETE</td>
|
||||
<td><code>/<bucket>/<key>?tagging</code></td>
|
||||
<td>Object tags.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>POST</td>
|
||||
<td><code>/<bucket>/<key>?uploads</code></td>
|
||||
<td>Initiate multipart upload.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>POST</td>
|
||||
<td><code>/<bucket>/<key>?select</code></td>
|
||||
<td>SQL query (SelectObjectContent).</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p class="small text-muted mt-3 mb-0">All responses include <code>X-Request-Id</code> for tracing. Logs land in <code>logs/api.log</code> and <code>logs/ui.log</code>.</p>
|
||||
<p class="small text-muted mt-3 mb-0">All responses include <code>X-Request-Id</code> for tracing. See the <a href="#api-matrix">Full API Reference</a> for the complete endpoint list. Logs land in <code>logs/api.log</code> and <code>logs/ui.log</code>.</p>
|
||||
</div>
|
||||
</article>
|
||||
<article id="examples" class="card shadow-sm docs-section">
|
||||
@@ -1310,6 +1427,10 @@ curl -X PUT "{{ api_base }}/bucket/<bucket>?quota" \
|
||||
<td><strong>KMS (SSE-KMS)</strong></td>
|
||||
<td>Encryption using customer-managed keys via the built-in KMS</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>SSE-C</strong></td>
|
||||
<td>Server-side encryption with customer-provided keys (per-request)</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
@@ -1376,6 +1497,54 @@ curl -X DELETE "{{ api_base }}/kms/keys/{key-id}?waiting_period_days=30" \
|
||||
<p class="small text-muted mb-0">
|
||||
<strong>Envelope Encryption:</strong> Each object is encrypted with a unique Data Encryption Key (DEK). The DEK is then encrypted (wrapped) by the master key or KMS key and stored alongside the ciphertext. On read, the DEK is unwrapped and used to decrypt the object transparently.
|
||||
</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">SSE-C (Customer-Provided Keys)</h3>
|
||||
<p class="small text-muted">With SSE-C, you supply your own 256-bit AES key with each request. The server encrypts/decrypts using your key but never stores it. You must provide the same key for both upload and download.</p>
|
||||
<div class="table-responsive mb-3">
|
||||
<table class="table table-sm table-bordered small">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Header</th>
|
||||
<th>Value</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>x-amz-server-side-encryption-customer-algorithm</code></td>
|
||||
<td><code>AES256</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>x-amz-server-side-encryption-customer-key</code></td>
|
||||
<td>Base64-encoded 256-bit key</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>x-amz-server-side-encryption-customer-key-MD5</code></td>
|
||||
<td>Base64-encoded MD5 of the key</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<pre class="mb-3"><code class="language-bash"># Generate a 256-bit key
|
||||
KEY=$(openssl rand -base64 32)
|
||||
KEY_MD5=$(echo -n "$KEY" | base64 -d | openssl dgst -md5 -binary | base64)
|
||||
|
||||
# Upload with SSE-C
|
||||
curl -X PUT "{{ api_base }}/my-bucket/secret.txt" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||||
-H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5" \
|
||||
--data-binary @secret.txt
|
||||
|
||||
# Download with SSE-C (same key required)
|
||||
curl "{{ api_base }}/my-bucket/secret.txt" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||||
-H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5"</code></pre>
|
||||
<div class="alert alert-light border mb-0 small">
|
||||
<strong>Note:</strong> SSE-C does not require <code>ENCRYPTION_ENABLED</code> or <code>KMS_ENABLED</code>. If you lose your key, the data is irrecoverable.
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article id="lifecycle" class="card shadow-sm docs-section">
|
||||
@@ -1460,10 +1629,217 @@ curl "{{ api_base }}/<bucket>?lifecycle" \
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article id="metrics" class="card shadow-sm docs-section">
|
||||
<article id="garbage-collection" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">14</span>
|
||||
<h2 class="h4 mb-0">Garbage Collection</h2>
|
||||
</div>
|
||||
<p class="text-muted">Automatically clean up orphaned data that accumulates over time: stale temp files, abandoned multipart uploads, stale lock files, orphaned metadata, orphaned versions, and empty directories.</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Enabling GC</h3>
|
||||
<p class="small text-muted">Disabled by default. Enable via environment variable:</p>
|
||||
<pre class="mb-3"><code class="language-bash">GC_ENABLED=true python run.py</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Configuration</h3>
|
||||
<div class="table-responsive mb-3">
|
||||
<table class="table table-sm table-bordered small">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Variable</th>
|
||||
<th>Default</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td><code>GC_ENABLED</code></td><td><code>false</code></td><td>Enable garbage collection</td></tr>
|
||||
<tr><td><code>GC_INTERVAL_HOURS</code></td><td><code>6</code></td><td>Hours between GC cycles</td></tr>
|
||||
<tr><td><code>GC_TEMP_FILE_MAX_AGE_HOURS</code></td><td><code>24</code></td><td>Delete temp files older than this</td></tr>
|
||||
<tr><td><code>GC_MULTIPART_MAX_AGE_DAYS</code></td><td><code>7</code></td><td>Delete orphaned multipart uploads older than this</td></tr>
|
||||
<tr><td><code>GC_LOCK_FILE_MAX_AGE_HOURS</code></td><td><code>1</code></td><td>Delete stale lock files older than this</td></tr>
|
||||
<tr><td><code>GC_DRY_RUN</code></td><td><code>false</code></td><td>Log what would be deleted without removing</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">What Gets Cleaned</h3>
|
||||
<div class="table-responsive mb-3">
|
||||
<table class="table table-sm table-bordered small">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Type</th>
|
||||
<th>Location</th>
|
||||
<th>Condition</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td><strong>Temp files</strong></td><td><code>.myfsio.sys/tmp/</code></td><td>Older than configured max age</td></tr>
|
||||
<tr><td><strong>Orphaned multipart</strong></td><td><code>.myfsio.sys/multipart/</code></td><td>Older than configured max age</td></tr>
|
||||
<tr><td><strong>Stale lock files</strong></td><td><code>.myfsio.sys/buckets/<bucket>/locks/</code></td><td>Older than configured max age</td></tr>
|
||||
<tr><td><strong>Orphaned metadata</strong></td><td><code>.myfsio.sys/buckets/<bucket>/meta/</code></td><td>Object file no longer exists</td></tr>
|
||||
<tr><td><strong>Orphaned versions</strong></td><td><code>.myfsio.sys/buckets/<bucket>/versions/</code></td><td>Main object no longer exists</td></tr>
|
||||
<tr><td><strong>Empty directories</strong></td><td>Various internal dirs</td><td>Directory is empty after cleanup</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Admin API</h3>
|
||||
<div class="table-responsive mb-3">
|
||||
<table class="table table-sm table-bordered small">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Method</th>
|
||||
<th>Route</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td><code>GET</code></td><td><code>/admin/gc/status</code></td><td>Get GC status and configuration</td></tr>
|
||||
<tr><td><code>POST</code></td><td><code>/admin/gc/run</code></td><td>Trigger manual GC run</td></tr>
|
||||
<tr><td><code>GET</code></td><td><code>/admin/gc/history</code></td><td>Get execution history</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<pre class="mb-3"><code class="language-bash"># Trigger a dry run (preview what would be cleaned)
|
||||
curl -X POST "{{ api_base }}/admin/gc/run" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"dry_run": true}'
|
||||
|
||||
# Trigger actual GC
|
||||
curl -X POST "{{ api_base }}/admin/gc/run" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# Check status
|
||||
curl "{{ api_base }}/admin/gc/status" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# View history
|
||||
curl "{{ api_base }}/admin/gc/history?limit=10" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||||
|
||||
<div class="alert alert-light border mb-0">
|
||||
<div class="d-flex gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||||
</svg>
|
||||
<div>
|
||||
<strong>Dry Run:</strong> Use <code>GC_DRY_RUN=true</code> or pass <code>{"dry_run": true}</code> to the API to preview what would be deleted without actually removing anything. Check the logs or API response for details.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article id="integrity" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">15</span>
|
||||
<h2 class="h4 mb-0">Integrity Scanner</h2>
|
||||
</div>
|
||||
<p class="text-muted">Detect and optionally auto-repair data inconsistencies: corrupted objects, orphaned files, phantom metadata, stale versions, ETag cache drift, and unmigrated legacy metadata.</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Enabling Integrity Scanner</h3>
|
||||
<p class="small text-muted">Disabled by default. Enable via environment variable:</p>
|
||||
<pre class="mb-3"><code class="language-bash">INTEGRITY_ENABLED=true python run.py</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Configuration</h3>
|
||||
<div class="table-responsive mb-3">
|
||||
<table class="table table-sm table-bordered small">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Variable</th>
|
||||
<th>Default</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td><code>INTEGRITY_ENABLED</code></td><td><code>false</code></td><td>Enable background integrity scanning</td></tr>
|
||||
<tr><td><code>INTEGRITY_INTERVAL_HOURS</code></td><td><code>24</code></td><td>Hours between scan cycles</td></tr>
|
||||
<tr><td><code>INTEGRITY_BATCH_SIZE</code></td><td><code>1000</code></td><td>Max objects to scan per cycle</td></tr>
|
||||
<tr><td><code>INTEGRITY_AUTO_HEAL</code></td><td><code>false</code></td><td>Automatically repair detected issues</td></tr>
|
||||
<tr><td><code>INTEGRITY_DRY_RUN</code></td><td><code>false</code></td><td>Log issues without healing</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">What Gets Checked</h3>
|
||||
<div class="table-responsive mb-3">
|
||||
<table class="table table-sm table-bordered small">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Check</th>
|
||||
<th>Detection</th>
|
||||
<th>Heal Action</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td><strong>Corrupted objects</strong></td><td>File MD5 does not match stored ETag</td><td>Update ETag in index (disk is authoritative)</td></tr>
|
||||
<tr><td><strong>Orphaned objects</strong></td><td>File exists without metadata entry</td><td>Create index entry with computed MD5/size/mtime</td></tr>
|
||||
<tr><td><strong>Phantom metadata</strong></td><td>Index entry exists but file is missing</td><td>Remove stale entry from index</td></tr>
|
||||
<tr><td><strong>Stale versions</strong></td><td>Manifest without data or vice versa</td><td>Remove orphaned version file</td></tr>
|
||||
<tr><td><strong>ETag cache</strong></td><td><code>etag_index.json</code> differs from metadata</td><td>Delete cache file (auto-rebuilt)</td></tr>
|
||||
<tr><td><strong>Legacy metadata</strong></td><td>Legacy <code>.meta.json</code> differs or unmigrated</td><td>Migrate to index, delete legacy file</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Admin API</h3>
|
||||
<div class="table-responsive mb-3">
|
||||
<table class="table table-sm table-bordered small">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Method</th>
|
||||
<th>Route</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td><code>GET</code></td><td><code>/admin/integrity/status</code></td><td>Get scanner status and configuration</td></tr>
|
||||
<tr><td><code>POST</code></td><td><code>/admin/integrity/run</code></td><td>Trigger manual scan</td></tr>
|
||||
<tr><td><code>GET</code></td><td><code>/admin/integrity/history</code></td><td>Get scan history</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<pre class="mb-3"><code class="language-bash"># Trigger a dry run with auto-heal preview
|
||||
curl -X POST "{{ api_base }}/admin/integrity/run" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"dry_run": true, "auto_heal": true}'
|
||||
|
||||
# Trigger actual scan with healing
|
||||
curl -X POST "{{ api_base }}/admin/integrity/run" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"auto_heal": true}'
|
||||
|
||||
# Check status
|
||||
curl "{{ api_base }}/admin/integrity/status" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# View history
|
||||
curl "{{ api_base }}/admin/integrity/history?limit=10" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||||
|
||||
<div class="alert alert-light border mb-0">
|
||||
<div class="d-flex gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||||
</svg>
|
||||
<div>
|
||||
<strong>Dry Run:</strong> Use <code>INTEGRITY_DRY_RUN=true</code> or pass <code>{"dry_run": true}</code> to the API to preview detected issues without making any changes. Combine with <code>{"auto_heal": true}</code> to see what would be repaired.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article id="metrics" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">16</span>
|
||||
<h2 class="h4 mb-0">Metrics History</h2>
|
||||
</div>
|
||||
<p class="text-muted">Track CPU, memory, and disk usage over time with optional metrics history. Disabled by default to minimize overhead.</p>
|
||||
@@ -1547,7 +1923,7 @@ curl -X PUT "{{ api_base | replace('/api', '/ui') }}/metrics/settings" \
|
||||
<article id="operation-metrics" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">15</span>
|
||||
<span class="docs-section-kicker">17</span>
|
||||
<h2 class="h4 mb-0">Operation Metrics</h2>
|
||||
</div>
|
||||
<p class="text-muted">Track API request statistics including request counts, latency, error rates, and bandwidth usage. Provides real-time visibility into API operations.</p>
|
||||
@@ -1654,7 +2030,7 @@ curl "{{ api_base | replace('/api', '/ui') }}/metrics/operations/history?hours=6
|
||||
<article id="troubleshooting" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">16</span>
|
||||
<span class="docs-section-kicker">18</span>
|
||||
<h2 class="h4 mb-0">Troubleshooting & tips</h2>
|
||||
</div>
|
||||
<div class="table-responsive">
|
||||
@@ -1705,7 +2081,7 @@ curl "{{ api_base | replace('/api', '/ui') }}/metrics/operations/history?hours=6
|
||||
<article id="health-check" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">17</span>
|
||||
<span class="docs-section-kicker">19</span>
|
||||
<h2 class="h4 mb-0">Health Check Endpoint</h2>
|
||||
</div>
|
||||
<p class="text-muted">The API exposes a health check endpoint for monitoring and load balancer integration.</p>
|
||||
@@ -1727,7 +2103,7 @@ curl {{ api_base }}/myfsio/health
|
||||
<article id="object-lock" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">18</span>
|
||||
<span class="docs-section-kicker">20</span>
|
||||
<h2 class="h4 mb-0">Object Lock & Retention</h2>
|
||||
</div>
|
||||
<p class="text-muted">Object Lock prevents objects from being deleted or overwritten for a specified retention period.</p>
|
||||
@@ -1787,7 +2163,7 @@ curl "{{ api_base }}/<bucket>/<key>?legal-hold" \
|
||||
<article id="access-logging" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">19</span>
|
||||
<span class="docs-section-kicker">21</span>
|
||||
<h2 class="h4 mb-0">Access Logging</h2>
|
||||
</div>
|
||||
<p class="text-muted">Enable S3-style access logging to track all requests to your buckets for audit and analysis.</p>
|
||||
@@ -1814,7 +2190,7 @@ curl "{{ api_base }}/<bucket>?logging" \
|
||||
<article id="notifications" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">20</span>
|
||||
<span class="docs-section-kicker">22</span>
|
||||
<h2 class="h4 mb-0">Notifications & Webhooks</h2>
|
||||
</div>
|
||||
<p class="text-muted">Configure event notifications to trigger webhooks when objects are created or deleted.</p>
|
||||
@@ -1877,7 +2253,7 @@ curl -X PUT "{{ api_base }}/<bucket>?notification" \
|
||||
<article id="select-content" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">21</span>
|
||||
<span class="docs-section-kicker">23</span>
|
||||
<h2 class="h4 mb-0">SelectObjectContent (SQL)</h2>
|
||||
</div>
|
||||
<p class="text-muted">Query CSV, JSON, or Parquet files directly using SQL without downloading the entire object.</p>
|
||||
@@ -1922,10 +2298,10 @@ curl -X POST "{{ api_base }}/<bucket>/data.csv?select" \
|
||||
<article id="advanced-ops" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">22</span>
|
||||
<span class="docs-section-kicker">24</span>
|
||||
<h2 class="h4 mb-0">Advanced S3 Operations</h2>
|
||||
</div>
|
||||
<p class="text-muted">Copy objects, upload part copies, and use range requests for partial downloads.</p>
|
||||
<p class="text-muted">Copy, move, and partially download objects using advanced S3 operations.</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">CopyObject</h3>
|
||||
<pre class="mb-3"><code class="language-bash"># Copy within same bucket
|
||||
@@ -1940,6 +2316,13 @@ curl -X PUT "{{ api_base }}/<bucket>/file.txt" \
|
||||
-H "x-amz-metadata-directive: REPLACE" \
|
||||
-H "x-amz-meta-newkey: newvalue"</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">MoveObject (UI)</h3>
|
||||
<p class="small text-muted">Move an object to a different key or bucket via the UI. Performs a copy then deletes the source. Requires <code>read</code>+<code>delete</code> on source and <code>write</code> on destination.</p>
|
||||
<pre class="mb-3"><code class="language-bash"># Move via UI API (session-authenticated)
|
||||
curl -X POST "http://localhost:5100/ui/buckets/<bucket>/objects/<key>/move" \
|
||||
-H "Content-Type: application/json" --cookie "session=..." \
|
||||
-d '{"dest_bucket": "other-bucket", "dest_key": "new-path/file.txt"}'</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">UploadPartCopy</h3>
|
||||
<p class="small text-muted">Copy data from an existing object into a multipart upload part:</p>
|
||||
<pre class="mb-3"><code class="language-bash"># Copy bytes 0-10485759 from source as part 1
|
||||
@@ -1989,7 +2372,7 @@ curl "{{ api_base }}/<bucket>/<key>" \
|
||||
<article id="acls" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">23</span>
|
||||
<span class="docs-section-kicker">25</span>
|
||||
<h2 class="h4 mb-0">Access Control Lists (ACLs)</h2>
|
||||
</div>
|
||||
<p class="text-muted">ACLs provide legacy-style permission management for buckets and objects.</p>
|
||||
@@ -2043,7 +2426,7 @@ curl -X PUT "{{ api_base }}/<bucket>/<key>" \
|
||||
<article id="tagging" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">24</span>
|
||||
<span class="docs-section-kicker">26</span>
|
||||
<h2 class="h4 mb-0">Object & Bucket Tagging</h2>
|
||||
</div>
|
||||
<p class="text-muted">Add metadata tags to buckets and objects for organization, cost allocation, or lifecycle rule filtering.</p>
|
||||
@@ -2099,6 +2482,367 @@ curl -X PUT "{{ api_base }}/<bucket>?tagging" \
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article id="website-hosting" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">27</span>
|
||||
<h2 class="h4 mb-0">Static Website Hosting</h2>
|
||||
</div>
|
||||
<p class="text-muted">Host static websites directly from S3 buckets with custom index and error pages, served via custom domain mapping.</p>
|
||||
|
||||
<div class="alert alert-info small mb-3">
|
||||
<strong>Prerequisite:</strong> Set <code>WEBSITE_HOSTING_ENABLED=true</code> to enable this feature.
|
||||
</div>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">1. Configure bucket for website hosting</h3>
|
||||
<pre class="mb-3"><code class="language-bash"># Enable website hosting with index and error documents
|
||||
curl -X PUT "{{ api_base }}/<bucket>?website" \
|
||||
-H "Content-Type: application/xml" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-d '<WebsiteConfiguration>
|
||||
<IndexDocument><Suffix>index.html</Suffix></IndexDocument>
|
||||
<ErrorDocument><Key>404.html</Key></ErrorDocument>
|
||||
</WebsiteConfiguration>'
|
||||
|
||||
# Get website configuration
|
||||
curl "{{ api_base }}/<bucket>?website" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# Remove website configuration
|
||||
curl -X DELETE "{{ api_base }}/<bucket>?website" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">2. Map a custom domain to the bucket</h3>
|
||||
<pre class="mb-3"><code class="language-bash"># Create domain mapping (admin only)
|
||||
curl -X POST "{{ api_base }}/admin/website-domains" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-d '{"domain": "example.com", "bucket": "my-site"}'
|
||||
|
||||
# List all domain mappings
|
||||
curl "{{ api_base }}/admin/website-domains" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# Update a mapping
|
||||
curl -X PUT "{{ api_base }}/admin/website-domains/example.com" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-d '{"bucket": "new-site-bucket"}'
|
||||
|
||||
# Delete a mapping
|
||||
curl -X DELETE "{{ api_base }}/admin/website-domains/example.com" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">3. Point your domain</h3>
|
||||
<p class="small text-muted">MyFSIO handles domain routing natively via the <code>Host</code> header — no path-based proxy rules needed. Just point your domain to the MyFSIO API server.</p>
|
||||
|
||||
<div class="alert alert-secondary small mb-3">
|
||||
<strong>Direct access (HTTP only):</strong> Point your domain's DNS (A or CNAME) directly to the MyFSIO server on port 5000.
|
||||
</div>
|
||||
|
||||
<p class="small text-muted mb-2">For <strong>HTTPS</strong>, place a reverse proxy in front. The proxy only needs to forward traffic — MyFSIO handles the domain-to-bucket routing:</p>
|
||||
<pre class="mb-3"><code class="language-nginx"># nginx example
|
||||
server {
|
||||
server_name example.com;
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:5000;
|
||||
proxy_set_header Host $host; # Required: passes the domain to MyFSIO
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}</code></pre>
|
||||
<div class="alert alert-warning small mb-3">
|
||||
<strong>Important:</strong> The <code>proxy_set_header Host $host;</code> directive is required. MyFSIO matches the incoming <code>Host</code> header against domain mappings to determine which bucket to serve.
|
||||
</div>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">How it works</h3>
|
||||
<div class="row g-2 mb-0">
|
||||
<div class="col-md-6">
|
||||
<ul class="small text-muted mb-0 ps-3">
|
||||
<li><code>/</code> serves the configured index document</li>
|
||||
<li><code>/about/</code> serves <code>about/index.html</code></li>
|
||||
<li>Objects served with correct Content-Type</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<ul class="small text-muted mb-0 ps-3">
|
||||
<li>Missing objects return the error document with 404</li>
|
||||
<li>Website endpoints are public (no auth required)</li>
|
||||
<li>Normal S3 API with auth continues to work</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article id="cors-config" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">28</span>
|
||||
<h2 class="h4 mb-0">CORS Configuration</h2>
|
||||
</div>
|
||||
<p class="text-muted">Configure per-bucket Cross-Origin Resource Sharing rules to control which origins can access your bucket from a browser.</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Setting CORS Rules</h3>
|
||||
<pre class="mb-3"><code class="language-bash"># Set CORS configuration
|
||||
curl -X PUT "{{ api_base }}/<bucket>?cors" \
|
||||
-H "Content-Type: application/xml" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-d '<CORSConfiguration>
|
||||
<CORSRule>
|
||||
<AllowedOrigin>https://example.com</AllowedOrigin>
|
||||
<AllowedMethod>GET</AllowedMethod>
|
||||
<AllowedMethod>PUT</AllowedMethod>
|
||||
<AllowedHeader>*</AllowedHeader>
|
||||
<ExposeHeader>ETag</ExposeHeader>
|
||||
<MaxAgeSeconds>3600</MaxAgeSeconds>
|
||||
</CORSRule>
|
||||
</CORSConfiguration>'
|
||||
|
||||
# Get CORS configuration
|
||||
curl "{{ api_base }}/<bucket>?cors" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# Delete CORS configuration
|
||||
curl -X DELETE "{{ api_base }}/<bucket>?cors" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Rule Fields</h3>
|
||||
<div class="table-responsive mb-0">
|
||||
<table class="table table-sm table-bordered small mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Field</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><code>AllowedOrigin</code></td>
|
||||
<td>Origins allowed to make requests (supports <code>*</code> wildcard)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>AllowedMethod</code></td>
|
||||
<td>HTTP methods: <code>GET</code>, <code>PUT</code>, <code>POST</code>, <code>DELETE</code>, <code>HEAD</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>AllowedHeader</code></td>
|
||||
<td>Request headers allowed in preflight (supports <code>*</code>)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>ExposeHeader</code></td>
|
||||
<td>Response headers visible to the browser (e.g., <code>ETag</code>, <code>x-amz-request-id</code>)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>MaxAgeSeconds</code></td>
|
||||
<td>How long the browser caches preflight results</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article id="post-object" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">29</span>
|
||||
<h2 class="h4 mb-0">PostObject (HTML Form Upload)</h2>
|
||||
</div>
|
||||
<p class="text-muted">Upload objects directly from an HTML form using browser-based POST uploads with policy-based authorization.</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Form Fields</h3>
|
||||
<div class="table-responsive mb-3">
|
||||
<table class="table table-sm table-bordered small">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Field</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td><code>key</code></td><td>Object key (supports <code>${filename}</code> variable)</td></tr>
|
||||
<tr><td><code>file</code></td><td>The file to upload</td></tr>
|
||||
<tr><td><code>policy</code></td><td>Base64-encoded policy document (JSON)</td></tr>
|
||||
<tr><td><code>x-amz-signature</code></td><td>HMAC-SHA256 signature of the policy</td></tr>
|
||||
<tr><td><code>x-amz-credential</code></td><td>Access key / date / region / s3 / aws4_request</td></tr>
|
||||
<tr><td><code>x-amz-algorithm</code></td><td><code>AWS4-HMAC-SHA256</code></td></tr>
|
||||
<tr><td><code>x-amz-date</code></td><td>ISO 8601 date (e.g., <code>20250101T000000Z</code>)</td></tr>
|
||||
<tr><td><code>Content-Type</code></td><td>MIME type of the uploaded file</td></tr>
|
||||
<tr><td><code>x-amz-meta-*</code></td><td>Custom metadata headers</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Simple Upload (No Signing)</h3>
|
||||
<pre class="mb-3"><code class="language-html"><form action="{{ api_base }}/my-bucket" method="POST" enctype="multipart/form-data">
|
||||
<input type="hidden" name="key" value="uploads/${filename}">
|
||||
<input type="file" name="file">
|
||||
<button type="submit">Upload</button>
|
||||
</form></code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Signed Upload (With Policy)</h3>
|
||||
<p class="small text-muted mb-0">For authenticated uploads, include a base64-encoded policy and SigV4 signature fields. The policy constrains allowed keys, content types, and size limits. See docs.md Section 20 for full signing examples.</p>
|
||||
</div>
|
||||
</article>
|
||||
<article id="list-objects-v2" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">30</span>
|
||||
<h2 class="h4 mb-0">List Objects API v2</h2>
|
||||
</div>
|
||||
<p class="text-muted">Use the v2 list API for improved pagination with continuation tokens instead of markers.</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Usage</h3>
|
||||
<pre class="mb-3"><code class="language-bash"># List with v2 API
|
||||
curl "{{ api_base }}/<bucket>?list-type=2&prefix=logs/&delimiter=/&max-keys=100" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# Paginate with continuation token
|
||||
curl "{{ api_base }}/<bucket>?list-type=2&continuation-token=<token>" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# Start listing after a specific key
|
||||
curl "{{ api_base }}/<bucket>?list-type=2&start-after=photos/2025/" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Query Parameters</h3>
|
||||
<div class="table-responsive mb-0">
|
||||
<table class="table table-sm table-bordered small mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Parameter</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td><code>list-type=2</code></td><td>Enables v2 API (required)</td></tr>
|
||||
<tr><td><code>prefix</code></td><td>Filter to keys starting with this prefix</td></tr>
|
||||
<tr><td><code>delimiter</code></td><td>Group keys by delimiter (typically <code>/</code> for folders)</td></tr>
|
||||
<tr><td><code>max-keys</code></td><td>Maximum objects to return (default 1000)</td></tr>
|
||||
<tr><td><code>continuation-token</code></td><td>Token from previous response for pagination</td></tr>
|
||||
<tr><td><code>start-after</code></td><td>Start listing after this key (first page only)</td></tr>
|
||||
<tr><td><code>fetch-owner</code></td><td>Include owner info in response</td></tr>
|
||||
<tr><td><code>encoding-type</code></td><td>Set to <code>url</code> to URL-encode keys in response</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article id="upgrading" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">31</span>
|
||||
<h2 class="h4 mb-0">Upgrading & Updates</h2>
|
||||
</div>
|
||||
<p class="text-muted">How to safely update MyFSIO to a new version.</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Pre-Update Backup</h3>
|
||||
<p class="small text-muted">Always back up before updating:</p>
|
||||
<pre class="mb-3"><code class="language-bash"># Back up configuration
|
||||
cp -r data/.myfsio.sys/config/ config-backup/
|
||||
|
||||
# Back up data (optional, for critical deployments)
|
||||
tar czf myfsio-backup-$(date +%Y%m%d).tar.gz data/
|
||||
|
||||
# Back up logs
|
||||
cp -r logs/ logs-backup/</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Update Procedure</h3>
|
||||
<ol class="docs-steps mb-3">
|
||||
<li><strong>Stop the service:</strong> <code>sudo systemctl stop myfsio</code> (or kill the process)</li>
|
||||
<li><strong>Pull new version:</strong> <code>git pull origin main</code> or download the new binary</li>
|
||||
<li><strong>Install dependencies:</strong> <code>pip install -r requirements.txt</code></li>
|
||||
<li><strong>Validate config:</strong> <code>python run.py --check-config</code></li>
|
||||
<li><strong>Start the service:</strong> <code>sudo systemctl start myfsio</code></li>
|
||||
<li><strong>Verify:</strong> <code>curl http://localhost:5000/myfsio/health</code></li>
|
||||
</ol>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Rollback</h3>
|
||||
<p class="small text-muted mb-0">If something goes wrong, stop the service, restore the backed-up config and data directories, then restart with the previous binary or code version. See <code>docs.md</code> Section 4 for detailed rollback procedures including blue-green deployment strategies.</p>
|
||||
</div>
|
||||
</article>
|
||||
<article id="api-matrix" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">32</span>
|
||||
<h2 class="h4 mb-0">Full API Reference</h2>
|
||||
</div>
|
||||
<p class="text-muted">Complete list of all S3-compatible, admin, and KMS endpoints.</p>
|
||||
<pre class="mb-0"><code class="language-text"># Service
|
||||
GET /myfsio/health # Health check
|
||||
|
||||
# Bucket Operations
|
||||
GET / # List buckets
|
||||
PUT /<bucket> # Create bucket
|
||||
DELETE /<bucket> # Delete bucket
|
||||
GET /<bucket> # List objects (?list-type=2)
|
||||
HEAD /<bucket> # Check bucket exists
|
||||
POST /<bucket> # POST object / form upload
|
||||
POST /<bucket>?delete # Bulk delete
|
||||
|
||||
# Bucket Configuration
|
||||
GET|PUT|DELETE /<bucket>?policy # Bucket policy
|
||||
GET|PUT /<bucket>?quota # Bucket quota
|
||||
GET|PUT /<bucket>?versioning # Versioning
|
||||
GET|PUT|DELETE /<bucket>?lifecycle # Lifecycle rules
|
||||
GET|PUT|DELETE /<bucket>?cors # CORS config
|
||||
GET|PUT|DELETE /<bucket>?encryption # Default encryption
|
||||
GET|PUT /<bucket>?acl # Bucket ACL
|
||||
GET|PUT|DELETE /<bucket>?tagging # Bucket tags
|
||||
GET|PUT|DELETE /<bucket>?replication # Replication rules
|
||||
GET|PUT /<bucket>?logging # Access logging
|
||||
GET|PUT /<bucket>?notification # Event notifications
|
||||
GET|PUT /<bucket>?object-lock # Object lock config
|
||||
GET|PUT|DELETE /<bucket>?website # Static website
|
||||
GET /<bucket>?uploads # List multipart uploads
|
||||
GET /<bucket>?versions # List object versions
|
||||
GET /<bucket>?location # Bucket region
|
||||
|
||||
# Object Operations
|
||||
PUT /<bucket>/<key> # Upload object
|
||||
GET /<bucket>/<key> # Download (Range supported)
|
||||
DELETE /<bucket>/<key> # Delete object
|
||||
HEAD /<bucket>/<key> # Object metadata
|
||||
POST /<bucket>/<key>?select # SQL query (SelectObjectContent)
|
||||
|
||||
# Object Configuration
|
||||
GET|PUT|DELETE /<bucket>/<key>?tagging # Object tags
|
||||
GET|PUT /<bucket>/<key>?acl # Object ACL
|
||||
GET|PUT /<bucket>/<key>?retention # Object retention
|
||||
GET|PUT /<bucket>/<key>?legal-hold # Legal hold
|
||||
|
||||
# Multipart Upload
|
||||
POST /<bucket>/<key>?uploads # Initiate
|
||||
PUT /<bucket>/<key>?uploadId=X&partNumber=N # Upload part
|
||||
POST /<bucket>/<key>?uploadId=X # Complete
|
||||
DELETE /<bucket>/<key>?uploadId=X # Abort
|
||||
GET /<bucket>/<key>?uploadId=X # List parts
|
||||
|
||||
# Copy (via x-amz-copy-source header)
|
||||
PUT /<bucket>/<key> # CopyObject
|
||||
PUT /<bucket>/<key>?uploadId&partNumber # UploadPartCopy
|
||||
|
||||
# Admin API
|
||||
GET|PUT /admin/site # Local site config
|
||||
GET /admin/sites # List peers
|
||||
POST /admin/sites # Register peer
|
||||
GET|PUT|DELETE /admin/sites/<id> # Manage peer
|
||||
GET /admin/sites/<id>/health # Peer health
|
||||
GET /admin/topology # Cluster topology
|
||||
GET|POST|PUT|DELETE /admin/website-domains # Domain mappings
|
||||
|
||||
# KMS API
|
||||
GET|POST /kms/keys # List / Create keys
|
||||
GET|DELETE /kms/keys/<id> # Get / Delete key
|
||||
POST /kms/keys/<id>/enable # Enable key
|
||||
POST /kms/keys/<id>/disable # Disable key
|
||||
POST /kms/keys/<id>/rotate # Rotate key
|
||||
POST /kms/encrypt # Encrypt data
|
||||
POST /kms/decrypt # Decrypt data
|
||||
POST /kms/generate-data-key # Generate data key
|
||||
POST /kms/generate-random # Generate random bytes</code></pre>
|
||||
</div>
|
||||
</article>
|
||||
</div>
|
||||
<div class="col-xl-4 docs-sidebar-col">
|
||||
<aside class="card shadow-sm docs-sidebar">
|
||||
@@ -2118,6 +2862,7 @@ curl -X PUT "{{ api_base }}/<bucket>?tagging" \
|
||||
<li><a href="#quotas">Bucket Quotas</a></li>
|
||||
<li><a href="#encryption">Encryption</a></li>
|
||||
<li><a href="#lifecycle">Lifecycle Rules</a></li>
|
||||
<li><a href="#garbage-collection">Garbage Collection</a></li>
|
||||
<li><a href="#metrics">Metrics History</a></li>
|
||||
<li><a href="#operation-metrics">Operation Metrics</a></li>
|
||||
<li><a href="#troubleshooting">Troubleshooting</a></li>
|
||||
@@ -2129,6 +2874,12 @@ curl -X PUT "{{ api_base }}/<bucket>?tagging" \
|
||||
<li><a href="#advanced-ops">Advanced Operations</a></li>
|
||||
<li><a href="#acls">Access Control Lists</a></li>
|
||||
<li><a href="#tagging">Object & Bucket Tagging</a></li>
|
||||
<li><a href="#website-hosting">Static Website Hosting</a></li>
|
||||
<li><a href="#cors-config">CORS Configuration</a></li>
|
||||
<li><a href="#post-object">PostObject (Form Upload)</a></li>
|
||||
<li><a href="#list-objects-v2">List Objects API v2</a></li>
|
||||
<li><a href="#upgrading">Upgrading & Updates</a></li>
|
||||
<li><a href="#api-matrix">Full API Reference</a></li>
|
||||
</ul>
|
||||
<div class="docs-sidebar-callouts">
|
||||
<div>
|
||||
|
||||
@@ -50,9 +50,20 @@
|
||||
New user created: <code>{{ disclosed_secret.access_key }}</code>
|
||||
{% endif %}
|
||||
</div>
|
||||
<p class="mb-2 small">⚠️ This secret is only shown once. Copy it now and store it securely.</p>
|
||||
<p class="mb-2 small">These credentials are only shown once. Copy them now and store them securely.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="input-group mb-2">
|
||||
<span class="input-group-text"><strong>Access key</strong></span>
|
||||
<input class="form-control font-monospace" type="text" value="{{ disclosed_secret.access_key }}" readonly id="disclosedAccessKeyValue" />
|
||||
<button class="btn btn-outline-primary" type="button" data-access-key-copy>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="bi bi-clipboard" viewBox="0 0 16 16">
|
||||
<path d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>
|
||||
<path d="M9.5 1a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1-.5-.5v-1a.5.5 0 0 1 .5-.5h3zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>
|
||||
</svg>
|
||||
Copy
|
||||
</button>
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<span class="input-group-text"><strong>Secret key</strong></span>
|
||||
<input class="form-control font-monospace" type="text" value="{{ disclosed_secret.secret_key }}" readonly id="disclosedSecretValue" />
|
||||
@@ -79,7 +90,7 @@
|
||||
<pre class="policy-preview mb-0" id="iamConfigPreview">{{ config_document }}</pre>
|
||||
<button class="btn btn-outline-light btn-sm config-copy" type="button" data-copy-target="iamConfigPreview">Copy JSON</button>
|
||||
</div>
|
||||
<p class="text-muted small mt-2 mb-0">Secrets are masked above. Access <code>{{ config_summary.path }}</code> directly to view full credentials.</p>
|
||||
<p class="text-muted small mt-2 mb-0">Secrets are masked above. IAM config is encrypted at rest.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -110,10 +121,34 @@
|
||||
{% else %}
|
||||
<div class="card-body px-4 pb-4">
|
||||
{% if users %}
|
||||
{% if users|length > 1 %}
|
||||
<div class="mb-3">
|
||||
<div class="search-input-wrapper">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="search-icon" viewBox="0 0 16 16">
|
||||
<path d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0z"/>
|
||||
</svg>
|
||||
<input type="text" class="form-control" id="iam-user-search" placeholder="Filter users by name or access key..." autocomplete="off" />
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="row g-3">
|
||||
{% for user in users %}
|
||||
<div class="col-md-6 col-xl-4">
|
||||
<div class="card h-100 iam-user-card">
|
||||
{% set ns = namespace(is_admin=false, is_expired=false, is_expiring_soon=false) %}
|
||||
{% for policy in user.policies %}
|
||||
{% if 'iam:*' in policy.actions or '*' in policy.actions %}
|
||||
{% set ns.is_admin = true %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if user.expires_at %}
|
||||
{% set exp_str = user.expires_at %}
|
||||
{% if exp_str <= now_iso %}
|
||||
{% set ns.is_expired = true %}
|
||||
{% elif exp_str <= soon_iso %}
|
||||
{% set ns.is_expiring_soon = true %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
<div class="col-md-6 col-xl-4 iam-user-item" data-display-name="{{ user.display_name|lower }}" data-access-key-filter="{{ user.access_key|lower }}">
|
||||
<div class="card h-100 iam-user-card{{ ' iam-admin-card' if ns.is_admin else '' }}">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-start justify-content-between mb-3">
|
||||
<div class="d-flex align-items-center gap-3 min-width-0 overflow-hidden">
|
||||
@@ -123,8 +158,28 @@
|
||||
</svg>
|
||||
</div>
|
||||
<div class="min-width-0">
|
||||
<div class="d-flex align-items-center gap-2 mb-0">
|
||||
<h6 class="fw-semibold mb-0 text-truncate" title="{{ user.display_name }}">{{ user.display_name }}</h6>
|
||||
<code class="small text-muted d-block text-truncate" title="{{ user.access_key }}">{{ user.access_key }}</code>
|
||||
{% if ns.is_admin %}
|
||||
<span class="iam-role-badge iam-role-admin" data-role-badge>Admin</span>
|
||||
{% else %}
|
||||
<span class="iam-role-badge iam-role-user" data-role-badge>User</span>
|
||||
{% endif %}
|
||||
{% if ns.is_expired %}
|
||||
<span class="badge text-bg-danger" style="font-size: .65rem">Expired</span>
|
||||
{% elif ns.is_expiring_soon %}
|
||||
<span class="badge text-bg-warning" style="font-size: .65rem">Expiring soon</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="d-flex align-items-center gap-1">
|
||||
<code class="small text-muted text-truncate" title="{{ user.access_key }}">{{ user.access_key }}</code>
|
||||
<button type="button" class="iam-copy-key" title="Copy access key" data-copy-access-key="{{ user.access_key }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>
|
||||
<path d="M9.5 1a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1-.5-.5v-1a.5.5 0 0 1 .5-.5h3zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="dropdown flex-shrink-0">
|
||||
@@ -142,6 +197,15 @@
|
||||
Edit Name
|
||||
</button>
|
||||
</li>
|
||||
<li>
|
||||
<button class="dropdown-item" type="button" data-expiry-user="{{ user.access_key }}" data-expires-at="{{ user.expires_at or '' }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
|
||||
<path d="M8 3.5a.5.5 0 0 0-1 0V9a.5.5 0 0 0 .252.434l3.5 2a.5.5 0 0 0 .496-.868L8 8.71V3.5z"/>
|
||||
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm7-8A7 7 0 1 1 1 8a7 7 0 0 1 14 0z"/>
|
||||
</svg>
|
||||
Set Expiry
|
||||
</button>
|
||||
</li>
|
||||
<li>
|
||||
<button class="dropdown-item" type="button" data-rotate-user="{{ user.access_key }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
|
||||
@@ -166,18 +230,27 @@
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<div class="small text-muted mb-2">Bucket Permissions</div>
|
||||
<div class="d-flex flex-wrap gap-1">
|
||||
<div class="d-flex flex-wrap gap-1" data-policy-badges>
|
||||
{% for policy in user.policies %}
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary">
|
||||
{% set bucket_label = 'All Buckets' if policy.bucket == '*' else policy.bucket %}
|
||||
{% if '*' in policy.actions %}
|
||||
{% set perm_label = 'Full Access' %}
|
||||
{% elif policy.actions|length >= 19 %}
|
||||
{% set perm_label = 'Full Access' %}
|
||||
{% elif 'list' in policy.actions and 'read' in policy.actions and 'write' in policy.actions and 'delete' in policy.actions %}
|
||||
{% set perm_label = 'Read + Write + Delete' %}
|
||||
{% elif 'list' in policy.actions and 'read' in policy.actions and 'write' in policy.actions %}
|
||||
{% set perm_label = 'Read + Write' %}
|
||||
{% elif 'list' in policy.actions and 'read' in policy.actions %}
|
||||
{% set perm_label = 'Read Only' %}
|
||||
{% else %}
|
||||
{% set perm_label = 'Custom (' ~ policy.actions|length ~ ')' %}
|
||||
{% endif %}
|
||||
<span class="iam-perm-badge">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>
|
||||
</svg>
|
||||
{{ policy.bucket }}
|
||||
{% if '*' in policy.actions %}
|
||||
<span class="opacity-75">(full)</span>
|
||||
{% else %}
|
||||
<span class="opacity-75">({{ policy.actions|length }})</span>
|
||||
{% endif %}
|
||||
{{ bucket_label }} · {{ perm_label }}
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="badge bg-secondary bg-opacity-10 text-secondary">No policies</span>
|
||||
@@ -196,6 +269,12 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div class="iam-no-results d-none" id="iam-no-results">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="mb-2" viewBox="0 0 16 16">
|
||||
<path d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0z"/>
|
||||
</svg>
|
||||
<p class="mb-0">No users match your filter.</p>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="empty-state text-center py-5">
|
||||
<div class="empty-state-icon mx-auto mb-3">
|
||||
@@ -237,6 +316,32 @@
|
||||
<label class="form-label fw-medium">Display Name</label>
|
||||
<input class="form-control" type="text" name="display_name" placeholder="Analytics Team" required autofocus />
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label class="form-label fw-medium d-flex justify-content-between align-items-center">
|
||||
Access Key <span class="text-muted fw-normal small">optional</span>
|
||||
</label>
|
||||
<div class="input-group">
|
||||
<input class="form-control font-monospace" type="text" name="access_key" id="createUserAccessKey" placeholder="Leave blank to auto-generate" />
|
||||
<button class="btn btn-outline-secondary" type="button" id="generateAccessKeyBtn" title="Generate secure access key">Generate</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label class="form-label fw-medium d-flex justify-content-between align-items-center">
|
||||
Secret Key <span class="text-muted fw-normal small">optional</span>
|
||||
</label>
|
||||
<div class="input-group">
|
||||
<input class="form-control font-monospace" type="text" name="secret_key" id="createUserSecretKey" placeholder="Leave blank to auto-generate" />
|
||||
<button class="btn btn-outline-secondary" type="button" id="generateSecretKeyBtn" title="Generate secure secret key">Generate</button>
|
||||
</div>
|
||||
<div class="form-text">If you set a custom secret key, copy it now. It will be encrypted and cannot be recovered.</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label class="form-label fw-medium d-flex justify-content-between align-items-center">
|
||||
Expiry <span class="text-muted fw-normal small">optional</span>
|
||||
</label>
|
||||
<input class="form-control" type="datetime-local" name="expires_at" id="createUserExpiry" />
|
||||
<div class="form-text">Leave blank for no expiration. Expired users cannot authenticate.</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label class="form-label fw-medium">Initial Policies (JSON)</label>
|
||||
<textarea class="form-control font-monospace" name="policies" id="createUserPolicies" rows="6" spellcheck="false" placeholder='[
|
||||
@@ -249,6 +354,8 @@
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-create-policy-template="full">Full Control</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-create-policy-template="readonly">Read-Only</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-create-policy-template="writer">Read + Write</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-create-policy-template="operator">Operator</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-create-policy-template="bucketadmin">Bucket Admin</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
@@ -299,6 +406,8 @@
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-policy-template="full">Full Control</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-policy-template="readonly">Read-Only</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-policy-template="writer">Read + Write</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-policy-template="operator">Operator</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-policy-template="bucketadmin">Bucket Admin</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
@@ -449,6 +558,52 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="expiryModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header border-0 pb-0">
|
||||
<h1 class="modal-title fs-5 fw-semibold">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M8 3.5a.5.5 0 0 0-1 0V9a.5.5 0 0 0 .252.434l3.5 2a.5.5 0 0 0 .496-.868L8 8.71V3.5z"/>
|
||||
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm7-8A7 7 0 1 1 1 8a7 7 0 0 1 14 0z"/>
|
||||
</svg>
|
||||
Set Expiry
|
||||
</h1>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form method="post" id="expiryForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
|
||||
<div class="modal-body">
|
||||
<p class="text-muted small mb-3">Set expiration for <code id="expiryUserLabel"></code></p>
|
||||
<div class="mb-3">
|
||||
<label class="form-label fw-medium">Expires at</label>
|
||||
<input class="form-control" type="datetime-local" name="expires_at" id="expiryDateInput" />
|
||||
<div class="form-text">Leave blank to remove expiration (never expires).</div>
|
||||
</div>
|
||||
<div class="d-flex flex-wrap gap-2">
|
||||
<span class="text-muted small me-2 align-self-center">Quick presets:</span>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-expiry-preset="1h">1 hour</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-expiry-preset="24h">24 hours</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-expiry-preset="7d">7 days</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-expiry-preset="30d">30 days</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-expiry-preset="90d">90 days</button>
|
||||
<button class="btn btn-outline-secondary btn-sm text-danger" type="button" data-expiry-preset="clear">Never</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button class="btn btn-primary" type="submit">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
|
||||
</svg>
|
||||
Save Expiry
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script id="iamUsersJson" type="application/json">{{ users | tojson }}</script>
|
||||
{% endblock %}
|
||||
|
||||
@@ -466,7 +621,8 @@
|
||||
updateUser: "{{ url_for('ui.update_iam_user', access_key='ACCESS_KEY') }}",
|
||||
deleteUser: "{{ url_for('ui.delete_iam_user', access_key='ACCESS_KEY') }}",
|
||||
updatePolicies: "{{ url_for('ui.update_iam_policies', access_key='ACCESS_KEY') }}",
|
||||
rotateSecret: "{{ url_for('ui.rotate_iam_secret', access_key='ACCESS_KEY') }}"
|
||||
rotateSecret: "{{ url_for('ui.rotate_iam_secret', access_key='ACCESS_KEY') }}",
|
||||
updateExpiry: "{{ url_for('ui.update_iam_expiry', access_key='ACCESS_KEY') }}"
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
@@ -73,9 +73,6 @@
|
||||
</svg>
|
||||
</button>
|
||||
</form>
|
||||
<div class="text-center mt-4">
|
||||
<small class="text-muted">Need help? Check the <a href="#" class="text-decoration-none">documentation</a></small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,7 +14,15 @@
|
||||
</h1>
|
||||
<p class="text-muted mb-0 mt-1">Configure this site's identity and manage peer sites for geo-distribution.</p>
|
||||
</div>
|
||||
<div class="d-none d-md-block">
|
||||
<div class="d-none d-md-flex align-items-center gap-2">
|
||||
{% if local_site and local_site.site_id %}
|
||||
<span class="badge bg-secondary bg-opacity-10 text-secondary fs-6 px-3 py-2">
|
||||
{{ local_site.site_id }}
|
||||
</span>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M1 8a.5.5 0 0 1 .5-.5h11.793l-3.147-3.146a.5.5 0 0 1 .708-.708l4 4a.5.5 0 0 1 0 .708l-4 4a.5.5 0 0 1-.708-.708L13.293 8.5H1.5A.5.5 0 0 1 1 8z"/>
|
||||
</svg>
|
||||
{% endif %}
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary fs-6 px-3 py-2">
|
||||
{{ peers|length }} peer{{ 's' if peers|length != 1 else '' }}
|
||||
</span>
|
||||
@@ -34,7 +42,7 @@
|
||||
<p class="text-muted small mb-0">This site's configuration</p>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
<form method="POST" action="{{ url_for('ui.update_local_site') }}">
|
||||
<form method="POST" action="{{ url_for('ui.update_local_site') }}" id="localSiteForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="mb-3">
|
||||
<label for="site_id" class="form-label fw-medium">Site ID</label>
|
||||
@@ -82,17 +90,25 @@
|
||||
</div>
|
||||
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<div class="card-header bg-transparent border-0 pt-3 pb-0 px-4">
|
||||
<button class="btn btn-link text-decoration-none p-0 w-100 d-flex align-items-center justify-content-between"
|
||||
type="button" data-bs-toggle="collapse" data-bs-target="#addPeerCollapse"
|
||||
aria-expanded="false" aria-controls="addPeerCollapse">
|
||||
<span class="d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 2a.5.5 0 0 1 .5.5v5h5a.5.5 0 0 1 0 1h-5v5a.5.5 0 0 1-1 0v-5h-5a.5.5 0 0 1 0-1h5v-5A.5.5 0 0 1 8 2Z"/>
|
||||
</svg>
|
||||
Add Peer Site
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Register a remote site</p>
|
||||
<span class="fw-semibold h5 mb-0">Add Peer Site</span>
|
||||
</span>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted add-peer-chevron" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<p class="text-muted small mb-0 mt-1">Register a remote site</p>
|
||||
</div>
|
||||
<div class="collapse" id="addPeerCollapse">
|
||||
<div class="card-body px-4 pb-4">
|
||||
<form method="POST" action="{{ url_for('ui.add_peer_site') }}">
|
||||
<form method="POST" action="{{ url_for('ui.add_peer_site') }}" id="addPeerForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="mb-3">
|
||||
<label for="peer_site_id" class="form-label fw-medium">Site ID</label>
|
||||
@@ -138,10 +154,11 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-8 col-md-7">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4 d-flex justify-content-between align-items-center">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4 d-flex justify-content-between align-items-start">
|
||||
<div>
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
@@ -151,6 +168,14 @@
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Known remote sites in the cluster</p>
|
||||
</div>
|
||||
{% if peers %}
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm" id="btnCheckAllHealth" title="Check health of all peers">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M11.251.068a.5.5 0 0 1 .227.58L9.677 6.5H13a.5.5 0 0 1 .364.843l-8 8.5a.5.5 0 0 1-.842-.49L6.323 9.5H3a.5.5 0 0 1-.364-.843l8-8.5a.5.5 0 0 1 .615-.09z"/>
|
||||
</svg>
|
||||
Check All
|
||||
</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
{% if peers %}
|
||||
@@ -172,7 +197,10 @@
|
||||
{% set peer = item.peer %}
|
||||
<tr data-site-id="{{ peer.site_id }}">
|
||||
<td class="text-center">
|
||||
<span class="peer-health-status" data-site-id="{{ peer.site_id }}" title="{% if peer.is_healthy == true %}Healthy{% elif peer.is_healthy == false %}Unhealthy{% else %}Unknown{% endif %}">
|
||||
<span class="peer-health-status" data-site-id="{{ peer.site_id }}"
|
||||
data-last-checked="{{ peer.last_health_check or '' }}"
|
||||
title="{% if peer.is_healthy == true %}Healthy{% elif peer.is_healthy == false %}Unhealthy{% else %}Not checked{% endif %}{% if peer.last_health_check %} (checked {{ peer.last_health_check }}){% endif %}"
|
||||
style="cursor: help;">
|
||||
{% if peer.is_healthy == true %}
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-success" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>
|
||||
@@ -205,64 +233,43 @@
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<span class="text-muted small text-truncate d-inline-block" style="max-width: 180px;" title="{{ peer.endpoint }}">{{ peer.endpoint }}</span>
|
||||
<span class="endpoint-display text-muted small" data-full-url="{{ peer.endpoint }}" title="{{ peer.endpoint }}" style="cursor: pointer;">
|
||||
{% set parsed = peer.endpoint.split('//') %}
|
||||
{% if parsed|length > 1 %}{{ parsed[1].split('/')[0] }}{% else %}{{ peer.endpoint }}{% endif %}
|
||||
</span>
|
||||
<button type="button" class="btn btn-link btn-sm p-0 ms-1 btn-copy-endpoint" data-url="{{ peer.endpoint }}" title="Copy full URL">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>
|
||||
<path d="M9.5 1a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1-.5-.5v-1a.5.5 0 0 1 .5-.5h3zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>
|
||||
</svg>
|
||||
</button>
|
||||
</td>
|
||||
<td><span class="badge bg-primary bg-opacity-10 text-primary">{{ peer.region }}</span></td>
|
||||
<td><span class="badge bg-secondary bg-opacity-10 text-secondary">{{ peer.priority }}</span></td>
|
||||
<td><span class="text-muted small">{{ peer.region }}</span></td>
|
||||
<td><span class="text-muted small">{{ peer.priority }}</span></td>
|
||||
<td class="sync-stats-cell" data-site-id="{{ peer.site_id }}">
|
||||
{% if item.has_connection %}
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary">{{ item.buckets_syncing }} bucket{{ 's' if item.buckets_syncing != 1 else '' }}</span>
|
||||
{% if item.has_bidirectional %}
|
||||
<span class="bidir-status-icon" data-site-id="{{ peer.site_id }}" title="Bidirectional sync configured - click to verify">
|
||||
<span class="bidir-status-icon" data-site-id="{{ peer.site_id }}" title="Bidirectional sync - click to verify">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="text-info" viewBox="0 0 16 16" style="cursor: pointer;">
|
||||
<path fill-rule="evenodd" d="M1 11.5a.5.5 0 0 0 .5.5h11.793l-3.147 3.146a.5.5 0 0 0 .708.708l4-4a.5.5 0 0 0 0-.708l-4-4a.5.5 0 0 0-.708.708L13.293 11H1.5a.5.5 0 0 0-.5.5zm14-7a.5.5 0 0 1-.5.5H2.707l3.147 3.146a.5.5 0 1 1-.708.708l-4-4a.5.5 0 0 1 0-.708l4-4a.5.5 0 1 1 .708.708L2.707 4H14.5a.5.5 0 0 1 .5.5z"/>
|
||||
</svg>
|
||||
</span>
|
||||
{% endif %}
|
||||
{% if item.buckets_syncing > 0 %}
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary btn-load-stats py-0 px-1"
|
||||
data-site-id="{{ peer.site_id }}" title="Load sync details">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>
|
||||
<path d="M8 4.466V.534a.25.25 0 0 1 .41-.192l2.36 1.966c.12.1.12.284 0 .384L8.41 4.658A.25.25 0 0 1 8 4.466z"/>
|
||||
</svg>
|
||||
</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="sync-stats-detail d-none mt-2 small" id="stats-{{ peer.site_id }}">
|
||||
<span class="spinner-border spinner-border-sm text-muted" style="width: 12px; height: 12px;"></span>
|
||||
</div>
|
||||
{% else %}
|
||||
<span class="text-muted small">No connection</span>
|
||||
<a href="#" class="text-muted small link-no-connection"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
title="Click to link a connection">Link a connection</a>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="text-end">
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
<a href="{{ url_for('ui.replication_wizard', site_id=peer.site_id) }}"
|
||||
class="btn btn-outline-primary {% if not item.has_connection %}disabled{% endif %}"
|
||||
title="Set up replication">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M11.534 7h3.932a.25.25 0 0 1 .192.41l-1.966 2.36a.25.25 0 0 1-.384 0l-1.966-2.36a.25.25 0 0 1 .192-.41zm-11 2h3.932a.25.25 0 0 0 .192-.41L2.692 6.23a.25.25 0 0 0-.384 0L.342 8.59A.25.25 0 0 0 .534 9z"/>
|
||||
<path fill-rule="evenodd" d="M8 3c-1.552 0-2.94.707-3.857 1.818a.5.5 0 1 1-.771-.636A6.002 6.002 0 0 1 13.917 7H12.9A5.002 5.002 0 0 0 8 3zM3.1 9a5.002 5.002 0 0 0 8.757 2.182.5.5 0 1 1 .771.636A6.002 6.002 0 0 1 2.083 9H3.1z"/>
|
||||
</svg>
|
||||
</a>
|
||||
<button type="button" class="btn btn-outline-info btn-check-bidir {% if not item.has_connection %}disabled{% endif %}"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
data-display-name="{{ peer.display_name or peer.site_id }}"
|
||||
title="Check bidirectional sync status">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M1 11.5a.5.5 0 0 0 .5.5h11.793l-3.147 3.146a.5.5 0 0 0 .708.708l4-4a.5.5 0 0 0 0-.708l-4-4a.5.5 0 0 0-.708.708L13.293 11H1.5a.5.5 0 0 0-.5.5zm14-7a.5.5 0 0 1-.5.5H2.707l3.147 3.146a.5.5 0 1 1-.708.708l-4-4a.5.5 0 0 1 0-.708l4-4a.5.5 0 1 1 .708.708L2.707 4H14.5a.5.5 0 0 1 .5.5z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-secondary btn-check-health"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
title="Check health">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M11.251.068a.5.5 0 0 1 .227.58L9.677 6.5H13a.5.5 0 0 1 .364.843l-8 8.5a.5.5 0 0 1-.842-.49L6.323 9.5H3a.5.5 0 0 1-.364-.843l8-8.5a.5.5 0 0 1 .615-.09z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-secondary"
|
||||
<div class="d-flex align-items-center justify-content-end gap-1">
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#editPeerModal"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
@@ -276,17 +283,68 @@
|
||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-danger"
|
||||
<div class="dropdown peer-actions-dropdown">
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-bs-toggle="dropdown" aria-expanded="false" title="More actions">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M3 9.5a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<ul class="dropdown-menu dropdown-menu-end">
|
||||
<li>
|
||||
<button type="button" class="dropdown-item btn-check-health" data-site-id="{{ peer.site_id }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-warning" viewBox="0 0 16 16">
|
||||
<path d="M11.251.068a.5.5 0 0 1 .227.58L9.677 6.5H13a.5.5 0 0 1 .364.843l-8 8.5a.5.5 0 0 1-.842-.49L6.323 9.5H3a.5.5 0 0 1-.364-.843l8-8.5a.5.5 0 0 1 .615-.09z"/>
|
||||
</svg>
|
||||
Check Health
|
||||
</button>
|
||||
</li>
|
||||
<li>
|
||||
<button type="button" class="dropdown-item btn-check-bidir {% if not item.has_connection %}disabled{% endif %}"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
data-display-name="{{ peer.display_name or peer.site_id }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-info" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M1 11.5a.5.5 0 0 0 .5.5h11.793l-3.147 3.146a.5.5 0 0 0 .708.708l4-4a.5.5 0 0 0 0-.708l-4-4a.5.5 0 0 0-.708.708L13.293 11H1.5a.5.5 0 0 0-.5.5zm14-7a.5.5 0 0 1-.5.5H2.707l3.147 3.146a.5.5 0 1 1-.708.708l-4-4a.5.5 0 0 1 0-.708l4-4a.5.5 0 1 1 .708.708L2.707 4H14.5a.5.5 0 0 1 .5.5z"/>
|
||||
</svg>
|
||||
Bidirectional Status
|
||||
</button>
|
||||
</li>
|
||||
{% if item.has_connection and item.buckets_syncing > 0 %}
|
||||
<li>
|
||||
<button type="button" class="dropdown-item btn-load-stats" data-site-id="{{ peer.site_id }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-primary" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>
|
||||
<path d="M8 4.466V.534a.25.25 0 0 1 .41-.192l2.36 1.966c.12.1.12.284 0 .384L8.41 4.658A.25.25 0 0 1 8 4.466z"/>
|
||||
</svg>
|
||||
Load Sync Stats
|
||||
</button>
|
||||
</li>
|
||||
{% endif %}
|
||||
<li>
|
||||
<a href="{{ url_for('ui.replication_wizard', site_id=peer.site_id) }}"
|
||||
class="dropdown-item {% if not item.has_connection %}disabled{% endif %}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-primary" viewBox="0 0 16 16">
|
||||
<path d="M11.534 7h3.932a.25.25 0 0 1 .192.41l-1.966 2.36a.25.25 0 0 1-.384 0l-1.966-2.36a.25.25 0 0 1 .192-.41zm-11 2h3.932a.25.25 0 0 0 .192-.41L2.692 6.23a.25.25 0 0 0-.384 0L.342 8.59A.25.25 0 0 0 .534 9z"/>
|
||||
<path fill-rule="evenodd" d="M8 3c-1.552 0-2.94.707-3.857 1.818a.5.5 0 1 1-.771-.636A6.002 6.002 0 0 1 13.917 7H12.9A5.002 5.002 0 0 0 8 3zM3.1 9a5.002 5.002 0 0 0 8.757 2.182.5.5 0 1 1 .771.636A6.002 6.002 0 0 1 2.083 9H3.1z"/>
|
||||
</svg>
|
||||
Replication Wizard
|
||||
</a>
|
||||
</li>
|
||||
<li><hr class="dropdown-divider"></li>
|
||||
<li>
|
||||
<button type="button" class="dropdown-item text-danger"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#deletePeerModal"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
data-display-name="{{ peer.display_name or peer.site_id }}"
|
||||
title="Delete peer">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
data-display-name="{{ peer.display_name or peer.site_id }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Delete Peer
|
||||
</button>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
@@ -445,99 +503,159 @@
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
const editPeerModal = document.getElementById('editPeerModal');
|
||||
var escapeHtml = window.UICore ? window.UICore.escapeHtml : function(s) { return s; };
|
||||
|
||||
var editPeerModal = document.getElementById('editPeerModal');
|
||||
if (editPeerModal) {
|
||||
editPeerModal.addEventListener('show.bs.modal', function (event) {
|
||||
const button = event.relatedTarget;
|
||||
const siteId = button.getAttribute('data-site-id');
|
||||
const endpoint = button.getAttribute('data-endpoint');
|
||||
const region = button.getAttribute('data-region');
|
||||
const priority = button.getAttribute('data-priority');
|
||||
const displayName = button.getAttribute('data-display-name');
|
||||
const connectionId = button.getAttribute('data-connection-id');
|
||||
|
||||
var button = event.relatedTarget;
|
||||
var siteId = button.getAttribute('data-site-id');
|
||||
document.getElementById('edit_site_id').value = siteId;
|
||||
document.getElementById('edit_endpoint').value = endpoint;
|
||||
document.getElementById('edit_region').value = region;
|
||||
document.getElementById('edit_priority').value = priority;
|
||||
document.getElementById('edit_display_name').value = displayName;
|
||||
document.getElementById('edit_connection_id').value = connectionId;
|
||||
|
||||
document.getElementById('edit_endpoint').value = button.getAttribute('data-endpoint');
|
||||
document.getElementById('edit_region').value = button.getAttribute('data-region');
|
||||
document.getElementById('edit_priority').value = button.getAttribute('data-priority');
|
||||
document.getElementById('edit_display_name').value = button.getAttribute('data-display-name');
|
||||
document.getElementById('edit_connection_id').value = button.getAttribute('data-connection-id');
|
||||
document.getElementById('editPeerForm').action = '/ui/sites/peers/' + encodeURIComponent(siteId) + '/update';
|
||||
});
|
||||
}
|
||||
|
||||
const deletePeerModal = document.getElementById('deletePeerModal');
|
||||
document.querySelectorAll('.link-no-connection').forEach(function(link) {
|
||||
link.addEventListener('click', function(e) {
|
||||
e.preventDefault();
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
var row = this.closest('tr[data-site-id]');
|
||||
if (row) {
|
||||
var btn = row.querySelector('.btn[data-bs-target="#editPeerModal"]');
|
||||
if (btn) btn.click();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
var deletePeerModal = document.getElementById('deletePeerModal');
|
||||
if (deletePeerModal) {
|
||||
deletePeerModal.addEventListener('show.bs.modal', function (event) {
|
||||
const button = event.relatedTarget;
|
||||
const siteId = button.getAttribute('data-site-id');
|
||||
const displayName = button.getAttribute('data-display-name');
|
||||
|
||||
var button = event.relatedTarget;
|
||||
var siteId = button.getAttribute('data-site-id');
|
||||
var displayName = button.getAttribute('data-display-name');
|
||||
document.getElementById('deletePeerName').textContent = displayName;
|
||||
document.getElementById('deletePeerForm').action = '/ui/sites/peers/' + encodeURIComponent(siteId) + '/delete';
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelectorAll('.btn-check-health').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
const siteId = this.getAttribute('data-site-id');
|
||||
const statusSpan = document.querySelector('.peer-health-status[data-site-id="' + siteId + '"]');
|
||||
function formatTimeAgo(date) {
|
||||
var seconds = Math.floor((new Date() - date) / 1000);
|
||||
if (seconds < 60) return 'just now';
|
||||
var minutes = Math.floor(seconds / 60);
|
||||
if (minutes < 60) return minutes + 'm ago';
|
||||
var hours = Math.floor(minutes / 60);
|
||||
if (hours < 24) return hours + 'h ago';
|
||||
return Math.floor(hours / 24) + 'd ago';
|
||||
}
|
||||
|
||||
function doHealthCheck(siteId) {
|
||||
var row = document.querySelector('tr[data-site-id="' + CSS.escape(siteId) + '"]');
|
||||
var statusSpan = row ? row.querySelector('.peer-health-status') : null;
|
||||
if (!statusSpan) return Promise.resolve();
|
||||
|
||||
statusSpan.innerHTML = '<span class="spinner-border spinner-border-sm text-muted" role="status" style="width: 14px; height: 14px;"></span>';
|
||||
|
||||
fetch('/ui/sites/peers/' + encodeURIComponent(siteId) + '/health')
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
return fetch('/ui/sites/peers/' + encodeURIComponent(siteId) + '/health')
|
||||
.then(function(response) { return response.json(); })
|
||||
.then(function(data) {
|
||||
var now = new Date();
|
||||
statusSpan.setAttribute('data-last-checked', now.toISOString());
|
||||
if (data.is_healthy) {
|
||||
statusSpan.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-success" viewBox="0 0 16 16"><path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/></svg>';
|
||||
statusSpan.title = 'Healthy';
|
||||
if (window.showToast) window.showToast('Peer site is healthy', 'Health Check', 'success');
|
||||
statusSpan.title = 'Healthy (checked ' + formatTimeAgo(now) + ')';
|
||||
return { siteId: siteId, healthy: true };
|
||||
} else {
|
||||
statusSpan.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-danger" viewBox="0 0 16 16"><path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/></svg>';
|
||||
statusSpan.title = 'Unhealthy' + (data.error ? ': ' + data.error : '');
|
||||
if (window.showToast) window.showToast(data.error || 'Peer site is unhealthy', 'Health Check', 'error');
|
||||
statusSpan.title = 'Unhealthy' + (data.error ? ': ' + data.error : '') + ' (checked ' + formatTimeAgo(now) + ')';
|
||||
return { siteId: siteId, healthy: false, error: data.error };
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
.catch(function(err) {
|
||||
statusSpan.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted" viewBox="0 0 16 16"><path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/><path d="M5.255 5.786a.237.237 0 0 0 .241.247h.825c.138 0 .248-.113.266-.25.09-.656.54-1.134 1.342-1.134.686 0 1.314.343 1.314 1.168 0 .635-.374.927-.965 1.371-.673.489-1.206 1.06-1.168 1.987l.003.217a.25.25 0 0 0 .25.246h.811a.25.25 0 0 0 .25-.25v-.105c0-.718.273-.927 1.01-1.486.609-.463 1.244-.977 1.244-2.056 0-1.511-1.276-2.241-2.673-2.241-1.267 0-2.655.59-2.75 2.286zm1.557 5.763c0 .533.425.927 1.01.927.609 0 1.028-.394 1.028-.927 0-.552-.42-.94-1.029-.94-.584 0-1.009.388-1.009.94z"/></svg>';
|
||||
statusSpan.title = 'Check failed';
|
||||
return { siteId: siteId, healthy: null };
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelectorAll('.btn-check-health').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
doHealthCheck(siteId).then(function(result) {
|
||||
if (!result) return;
|
||||
if (result.healthy === true) {
|
||||
if (window.showToast) window.showToast('Peer site is healthy', 'Health Check', 'success');
|
||||
} else if (result.healthy === false) {
|
||||
if (window.showToast) window.showToast(result.error || 'Peer site is unhealthy', 'Health Check', 'error');
|
||||
} else {
|
||||
if (window.showToast) window.showToast('Failed to check health', 'Health Check', 'error');
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
var checkAllBtn = document.getElementById('btnCheckAllHealth');
|
||||
if (checkAllBtn) {
|
||||
checkAllBtn.addEventListener('click', function() {
|
||||
var btn = this;
|
||||
var originalHtml = btn.innerHTML;
|
||||
btn.disabled = true;
|
||||
btn.innerHTML = '<span class="spinner-border spinner-border-sm me-1"></span>Checking...';
|
||||
|
||||
var siteIds = [];
|
||||
document.querySelectorAll('.peer-health-status').forEach(function(el) {
|
||||
siteIds.push(el.getAttribute('data-site-id'));
|
||||
});
|
||||
|
||||
var promises = siteIds.map(function(id) { return doHealthCheck(id); });
|
||||
Promise.all(promises).then(function(results) {
|
||||
var healthy = results.filter(function(r) { return r && r.healthy === true; }).length;
|
||||
var unhealthy = results.filter(function(r) { return r && r.healthy === false; }).length;
|
||||
var failed = results.filter(function(r) { return r && r.healthy === null; }).length;
|
||||
|
||||
var msg = healthy + ' healthy';
|
||||
if (unhealthy > 0) msg += ', ' + unhealthy + ' unhealthy';
|
||||
if (failed > 0) msg += ', ' + failed + ' failed';
|
||||
if (window.showToast) window.showToast(msg, 'Health Check', unhealthy > 0 ? 'warning' : 'success');
|
||||
|
||||
btn.disabled = false;
|
||||
btn.innerHTML = originalHtml;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelectorAll('.btn-load-stats').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
const siteId = this.getAttribute('data-site-id');
|
||||
const detailDiv = document.getElementById('stats-' + siteId);
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
var detailDiv = document.getElementById('stats-' + siteId);
|
||||
if (!detailDiv) return;
|
||||
|
||||
detailDiv.classList.remove('d-none');
|
||||
detailDiv.innerHTML = '<span class="spinner-border spinner-border-sm text-muted" style="width: 12px; height: 12px;"></span> Loading...';
|
||||
|
||||
fetch('/ui/sites/peers/' + encodeURIComponent(siteId) + '/sync-stats')
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
.then(function(response) { return response.json(); })
|
||||
.then(function(data) {
|
||||
if (data.error) {
|
||||
detailDiv.innerHTML = '<span class="text-danger">' + data.error + '</span>';
|
||||
detailDiv.innerHTML = '<span class="text-danger">' + escapeHtml(data.error) + '</span>';
|
||||
} else {
|
||||
const lastSync = data.last_sync_at
|
||||
var lastSync = data.last_sync_at
|
||||
? new Date(data.last_sync_at * 1000).toLocaleString()
|
||||
: 'Never';
|
||||
detailDiv.innerHTML = `
|
||||
<div class="d-flex flex-wrap gap-2 mb-1">
|
||||
<span class="text-success"><strong>${data.objects_synced}</strong> synced</span>
|
||||
<span class="text-warning"><strong>${data.objects_pending}</strong> pending</span>
|
||||
<span class="text-danger"><strong>${data.objects_failed}</strong> failed</span>
|
||||
</div>
|
||||
<div class="text-muted" style="font-size: 0.75rem;">
|
||||
Last sync: ${lastSync}
|
||||
</div>
|
||||
`;
|
||||
detailDiv.innerHTML =
|
||||
'<div class="d-flex flex-wrap gap-2 mb-1">' +
|
||||
'<span class="text-success"><strong>' + escapeHtml(String(data.objects_synced)) + '</strong> synced</span>' +
|
||||
'<span class="text-warning"><strong>' + escapeHtml(String(data.objects_pending)) + '</strong> pending</span>' +
|
||||
'<span class="text-danger"><strong>' + escapeHtml(String(data.objects_failed)) + '</strong> failed</span>' +
|
||||
'</div>' +
|
||||
'<div class="text-muted" style="font-size: 0.75rem;">Last sync: ' + escapeHtml(lastSync) + '</div>';
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
.catch(function() {
|
||||
detailDiv.innerHTML = '<span class="text-danger">Failed to load stats</span>';
|
||||
});
|
||||
});
|
||||
@@ -545,181 +663,117 @@
|
||||
|
||||
document.querySelectorAll('.bidir-status-icon').forEach(function(icon) {
|
||||
icon.addEventListener('click', function() {
|
||||
const siteId = this.getAttribute('data-site-id');
|
||||
const btn = document.querySelector('.btn-check-bidir[data-site-id="' + siteId + '"]');
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
var row = this.closest('tr[data-site-id]');
|
||||
var btn = row ? row.querySelector('.btn-check-bidir') : null;
|
||||
if (btn) btn.click();
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.btn-check-bidir').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
const siteId = this.getAttribute('data-site-id');
|
||||
const displayName = this.getAttribute('data-display-name');
|
||||
const modal = new bootstrap.Modal(document.getElementById('bidirStatusModal'));
|
||||
const contentDiv = document.getElementById('bidirStatusContent');
|
||||
const wizardLink = document.getElementById('bidirWizardLink');
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
var displayName = this.getAttribute('data-display-name');
|
||||
var modal = new bootstrap.Modal(document.getElementById('bidirStatusModal'));
|
||||
var contentDiv = document.getElementById('bidirStatusContent');
|
||||
var wizardLink = document.getElementById('bidirWizardLink');
|
||||
|
||||
contentDiv.innerHTML = `
|
||||
<div class="text-center py-4">
|
||||
<span class="spinner-border text-primary" role="status"></span>
|
||||
<p class="text-muted mt-2 mb-0">Checking bidirectional configuration with ${displayName}...</p>
|
||||
</div>
|
||||
`;
|
||||
contentDiv.innerHTML =
|
||||
'<div class="text-center py-4">' +
|
||||
'<span class="spinner-border text-primary" role="status"></span>' +
|
||||
'<p class="text-muted mt-2 mb-0">Checking bidirectional configuration with ' + escapeHtml(displayName) + '...</p>' +
|
||||
'</div>';
|
||||
wizardLink.classList.add('d-none');
|
||||
modal.show();
|
||||
|
||||
fetch('/ui/sites/peers/' + encodeURIComponent(siteId) + '/bidirectional-status')
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
let html = '';
|
||||
.then(function(response) { return response.json(); })
|
||||
.then(function(data) {
|
||||
var html = '';
|
||||
|
||||
if (data.is_fully_configured) {
|
||||
html += `
|
||||
<div class="alert alert-success d-flex align-items-center mb-4" role="alert">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="flex-shrink-0 me-2" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>
|
||||
</svg>
|
||||
<div>
|
||||
<strong>Bidirectional sync is fully configured!</strong><br>
|
||||
<small>Both sites are set up to sync data in both directions.</small>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
html += '<div class="alert alert-success d-flex align-items-center mb-4" role="alert">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="flex-shrink-0 me-2" viewBox="0 0 16 16">' +
|
||||
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>' +
|
||||
'</svg>' +
|
||||
'<div><strong>Bidirectional sync is fully configured!</strong><br><small>Both sites are set up to sync data in both directions.</small></div>' +
|
||||
'</div>';
|
||||
} else if (data.issues && data.issues.length > 0) {
|
||||
const errors = data.issues.filter(i => i.severity === 'error');
|
||||
const warnings = data.issues.filter(i => i.severity === 'warning');
|
||||
var errors = data.issues.filter(function(i) { return i.severity === 'error'; });
|
||||
var warnings = data.issues.filter(function(i) { return i.severity === 'warning'; });
|
||||
|
||||
if (errors.length > 0) {
|
||||
html += `
|
||||
<div class="alert alert-danger mb-3" role="alert">
|
||||
<h6 class="alert-heading fw-bold mb-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>
|
||||
</svg>
|
||||
Configuration Errors
|
||||
</h6>
|
||||
<ul class="mb-0 ps-3">
|
||||
`;
|
||||
errors.forEach(issue => {
|
||||
html += `<li><strong>${issue.code}:</strong> ${issue.message}</li>`;
|
||||
html += '<div class="alert alert-danger mb-3" role="alert">' +
|
||||
'<h6 class="alert-heading fw-bold mb-2">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16"><path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/></svg>' +
|
||||
' Configuration Errors</h6><ul class="mb-0 ps-3">';
|
||||
errors.forEach(function(issue) {
|
||||
html += '<li><strong>' + escapeHtml(issue.code) + ':</strong> ' + escapeHtml(issue.message) + '</li>';
|
||||
});
|
||||
html += '</ul></div>';
|
||||
}
|
||||
|
||||
if (warnings.length > 0) {
|
||||
html += `
|
||||
<div class="alert alert-warning mb-3" role="alert">
|
||||
<h6 class="alert-heading fw-bold mb-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/>
|
||||
</svg>
|
||||
Warnings
|
||||
</h6>
|
||||
<ul class="mb-0 ps-3">
|
||||
`;
|
||||
warnings.forEach(issue => {
|
||||
html += `<li><strong>${issue.code}:</strong> ${issue.message}</li>`;
|
||||
html += '<div class="alert alert-warning mb-3" role="alert">' +
|
||||
'<h6 class="alert-heading fw-bold mb-2">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16"><path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/></svg>' +
|
||||
' Warnings</h6><ul class="mb-0 ps-3">';
|
||||
warnings.forEach(function(issue) {
|
||||
html += '<li><strong>' + escapeHtml(issue.code) + ':</strong> ' + escapeHtml(issue.message) + '</li>';
|
||||
});
|
||||
html += '</ul></div>';
|
||||
}
|
||||
}
|
||||
|
||||
html += '<div class="row g-3">';
|
||||
|
||||
html += `
|
||||
<div class="col-md-6">
|
||||
<div class="card h-100">
|
||||
<div class="card-header bg-light py-2">
|
||||
<strong>This Site (Local)</strong>
|
||||
</div>
|
||||
<div class="card-body small">
|
||||
<p class="mb-1"><strong>Site ID:</strong> ${data.local_site_id || '<span class="text-danger">Not configured</span>'}</p>
|
||||
<p class="mb-1"><strong>Endpoint:</strong> ${data.local_endpoint || '<span class="text-danger">Not configured</span>'}</p>
|
||||
<p class="mb-1"><strong>Site Sync Worker:</strong> ${data.local_site_sync_enabled ? '<span class="text-success">Enabled</span>' : '<span class="text-warning">Disabled</span>'}</p>
|
||||
<p class="mb-0"><strong>Bidirectional Rules:</strong> ${data.local_bidirectional_rules ? data.local_bidirectional_rules.length : 0}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
html += '<div class="col-md-6"><div class="card h-100"><div class="card-header bg-light py-2"><strong>This Site (Local)</strong></div>' +
|
||||
'<div class="card-body small">' +
|
||||
'<p class="mb-1"><strong>Site ID:</strong> ' + (data.local_site_id ? escapeHtml(data.local_site_id) : '<span class="text-danger">Not configured</span>') + '</p>' +
|
||||
'<p class="mb-1"><strong>Endpoint:</strong> ' + (data.local_endpoint ? escapeHtml(data.local_endpoint) : '<span class="text-danger">Not configured</span>') + '</p>' +
|
||||
'<p class="mb-1"><strong>Site Sync Worker:</strong> ' + (data.local_site_sync_enabled ? '<span class="text-success">Enabled</span>' : '<span class="text-warning">Disabled</span>') + '</p>' +
|
||||
'<p class="mb-0"><strong>Bidirectional Rules:</strong> ' + (data.local_bidirectional_rules ? data.local_bidirectional_rules.length : 0) + '</p>' +
|
||||
'</div></div></div>';
|
||||
|
||||
if (data.remote_status) {
|
||||
const rs = data.remote_status;
|
||||
html += `
|
||||
<div class="col-md-6">
|
||||
<div class="card h-100">
|
||||
<div class="card-header bg-light py-2">
|
||||
<strong>Remote Site (${displayName})</strong>
|
||||
</div>
|
||||
<div class="card-body small">
|
||||
`;
|
||||
var rs = data.remote_status;
|
||||
html += '<div class="col-md-6"><div class="card h-100"><div class="card-header bg-light py-2"><strong>Remote Site (' + escapeHtml(displayName) + ')</strong></div>' +
|
||||
'<div class="card-body small">';
|
||||
if (rs.admin_access_denied) {
|
||||
html += '<p class="text-warning mb-0">Admin access denied - cannot verify remote configuration</p>';
|
||||
} else if (rs.reachable === false) {
|
||||
html += '<p class="text-danger mb-0">Could not reach remote admin API</p>';
|
||||
} else {
|
||||
html += `
|
||||
<p class="mb-1"><strong>Has Peer Entry for Us:</strong> ${rs.has_peer_for_us ? '<span class="text-success">Yes</span>' : '<span class="text-danger">No</span>'}</p>
|
||||
<p class="mb-1"><strong>Connection Configured:</strong> ${rs.peer_connection_configured ? '<span class="text-success">Yes</span>' : '<span class="text-danger">No</span>'}</p>
|
||||
`;
|
||||
html += '<p class="mb-1"><strong>Has Peer Entry for Us:</strong> ' + (rs.has_peer_for_us ? '<span class="text-success">Yes</span>' : '<span class="text-danger">No</span>') + '</p>' +
|
||||
'<p class="mb-1"><strong>Connection Configured:</strong> ' + (rs.peer_connection_configured ? '<span class="text-success">Yes</span>' : '<span class="text-danger">No</span>') + '</p>';
|
||||
}
|
||||
html += '</div></div></div>';
|
||||
} else {
|
||||
html += `
|
||||
<div class="col-md-6">
|
||||
<div class="card h-100">
|
||||
<div class="card-header bg-light py-2">
|
||||
<strong>Remote Site (${displayName})</strong>
|
||||
</div>
|
||||
<div class="card-body small">
|
||||
<p class="text-muted mb-0">Could not check remote status</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
html += '<div class="col-md-6"><div class="card h-100"><div class="card-header bg-light py-2"><strong>Remote Site (' + escapeHtml(displayName) + ')</strong></div>' +
|
||||
'<div class="card-body small"><p class="text-muted mb-0">Could not check remote status</p></div></div></div>';
|
||||
}
|
||||
|
||||
html += '</div>';
|
||||
|
||||
if (data.local_bidirectional_rules && data.local_bidirectional_rules.length > 0) {
|
||||
html += `
|
||||
<div class="mt-3">
|
||||
<h6 class="fw-semibold">Local Bidirectional Rules</h6>
|
||||
<table class="table table-sm table-bordered mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Source Bucket</th>
|
||||
<th>Target Bucket</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
`;
|
||||
data.local_bidirectional_rules.forEach(rule => {
|
||||
html += `
|
||||
<tr>
|
||||
<td>${rule.bucket_name}</td>
|
||||
<td>${rule.target_bucket}</td>
|
||||
<td>${rule.enabled ? '<span class="badge bg-success">Enabled</span>' : '<span class="badge bg-secondary">Disabled</span>'}</td>
|
||||
</tr>
|
||||
`;
|
||||
html += '<div class="mt-3"><h6 class="fw-semibold">Local Bidirectional Rules</h6>' +
|
||||
'<table class="table table-sm table-bordered mb-0"><thead class="table-light"><tr><th>Source Bucket</th><th>Target Bucket</th><th>Status</th></tr></thead><tbody>';
|
||||
data.local_bidirectional_rules.forEach(function(rule) {
|
||||
html += '<tr><td>' + escapeHtml(rule.bucket_name) + '</td><td>' + escapeHtml(rule.target_bucket) + '</td>' +
|
||||
'<td>' + (rule.enabled ? '<span class="badge bg-success">Enabled</span>' : '<span class="badge bg-secondary">Disabled</span>') + '</td></tr>';
|
||||
});
|
||||
html += '</tbody></table></div>';
|
||||
}
|
||||
|
||||
if (!data.is_fully_configured) {
|
||||
html += `
|
||||
<div class="alert alert-info mt-3 mb-0" role="alert">
|
||||
<h6 class="alert-heading fw-bold">How to Fix</h6>
|
||||
<ol class="mb-0 ps-3">
|
||||
<li>Ensure this site has a Site ID and Endpoint URL configured</li>
|
||||
<li>On the remote site, register this site as a peer with a connection</li>
|
||||
<li>Create bidirectional replication rules on both sites</li>
|
||||
<li>Enable SITE_SYNC_ENABLED=true on both sites</li>
|
||||
</ol>
|
||||
</div>
|
||||
`;
|
||||
const blockingErrors = ['NO_CONNECTION', 'CONNECTION_NOT_FOUND', 'REMOTE_UNREACHABLE', 'ENDPOINT_NOT_ALLOWED'];
|
||||
const hasBlockingError = data.issues && data.issues.some(i => blockingErrors.includes(i.code));
|
||||
html += '<div class="alert alert-info mt-3 mb-0" role="alert">' +
|
||||
'<h6 class="alert-heading fw-bold">How to Fix</h6>' +
|
||||
'<ol class="mb-0 ps-3">' +
|
||||
'<li>Ensure this site has a Site ID and Endpoint URL configured</li>' +
|
||||
'<li>On the remote site, register this site as a peer with a connection</li>' +
|
||||
'<li>Create bidirectional replication rules on both sites</li>' +
|
||||
'<li>Enable SITE_SYNC_ENABLED=true on both sites</li>' +
|
||||
'</ol></div>';
|
||||
var blockingErrors = ['NO_CONNECTION', 'CONNECTION_NOT_FOUND', 'REMOTE_UNREACHABLE', 'ENDPOINT_NOT_ALLOWED'];
|
||||
var hasBlockingError = data.issues && data.issues.some(function(i) { return blockingErrors.indexOf(i.code) !== -1; });
|
||||
if (!hasBlockingError) {
|
||||
wizardLink.href = '/ui/sites/peers/' + encodeURIComponent(siteId) + '/replication-wizard';
|
||||
wizardLink.classList.remove('d-none');
|
||||
@@ -728,15 +782,110 @@
|
||||
|
||||
contentDiv.innerHTML = html;
|
||||
})
|
||||
.catch(err => {
|
||||
contentDiv.innerHTML = `
|
||||
<div class="alert alert-danger" role="alert">
|
||||
<strong>Error:</strong> Failed to check bidirectional status. ${err.message || ''}
|
||||
</div>
|
||||
`;
|
||||
.catch(function(err) {
|
||||
contentDiv.innerHTML = '<div class="alert alert-danger" role="alert"><strong>Error:</strong> Failed to check bidirectional status. ' + escapeHtml(err.message || '') + '</div>';
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.btn-copy-endpoint').forEach(function(btn) {
|
||||
btn.addEventListener('click', function(e) {
|
||||
e.stopPropagation();
|
||||
var url = this.getAttribute('data-url');
|
||||
if (window.UICore && window.UICore.copyToClipboard) {
|
||||
window.UICore.copyToClipboard(url).then(function(ok) {
|
||||
if (ok && window.showToast) window.showToast('Endpoint URL copied', 'Copied', 'success');
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
var localSiteForm = document.getElementById('localSiteForm');
|
||||
if (localSiteForm) {
|
||||
localSiteForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(this, {
|
||||
successMessage: 'Local site configuration updated',
|
||||
onSuccess: function() {
|
||||
setTimeout(function() { location.reload(); }, 800);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var addPeerForm = document.getElementById('addPeerForm');
|
||||
if (addPeerForm) {
|
||||
addPeerForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(this, {
|
||||
successMessage: 'Peer site added',
|
||||
onSuccess: function(data) {
|
||||
if (data.redirect) {
|
||||
setTimeout(function() { window.location.href = data.redirect; }, 800);
|
||||
} else {
|
||||
setTimeout(function() { location.reload(); }, 800);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var editPeerForm = document.getElementById('editPeerForm');
|
||||
if (editPeerForm) {
|
||||
editPeerForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
var modal = bootstrap.Modal.getInstance(document.getElementById('editPeerModal'));
|
||||
window.UICore.submitFormAjax(this, {
|
||||
successMessage: 'Peer site updated',
|
||||
onSuccess: function() {
|
||||
if (modal) modal.hide();
|
||||
setTimeout(function() { location.reload(); }, 800);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var deletePeerForm = document.getElementById('deletePeerForm');
|
||||
if (deletePeerForm) {
|
||||
deletePeerForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
var modal = bootstrap.Modal.getInstance(document.getElementById('deletePeerModal'));
|
||||
window.UICore.submitFormAjax(this, {
|
||||
successMessage: 'Peer site deleted',
|
||||
onSuccess: function() {
|
||||
if (modal) modal.hide();
|
||||
setTimeout(function() { location.reload(); }, 800);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
document.querySelectorAll('.peer-actions-dropdown').forEach(function(dd) {
|
||||
dd.addEventListener('shown.bs.dropdown', function() {
|
||||
var toggle = dd.querySelector('[data-bs-toggle="dropdown"]');
|
||||
var menu = dd.querySelector('.dropdown-menu');
|
||||
if (!toggle || !menu) return;
|
||||
var rect = toggle.getBoundingClientRect();
|
||||
menu.style.top = rect.bottom + 'px';
|
||||
menu.style.left = (rect.right - menu.offsetWidth) + 'px';
|
||||
});
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
|
||||
<style>
|
||||
.add-peer-chevron {
|
||||
transition: transform 0.2s ease;
|
||||
}
|
||||
[aria-expanded="true"] .add-peer-chevron {
|
||||
transform: rotate(180deg);
|
||||
}
|
||||
.endpoint-display:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
.peer-actions-dropdown .dropdown-menu {
|
||||
position: fixed !important;
|
||||
inset: auto !important;
|
||||
transform: none !important;
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
750
templates/system.html
Normal file
750
templates/system.html
Normal file
@@ -0,0 +1,750 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}System - MyFSIO Console{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="page-header d-flex justify-content-between align-items-center mb-4">
|
||||
<div>
|
||||
<p class="text-uppercase text-muted small mb-1">Administration</p>
|
||||
<h1 class="h3 mb-1 d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M9.405 1.05c-.413-1.4-2.397-1.4-2.81 0l-.1.34a1.464 1.464 0 0 1-2.105.872l-.31-.17c-1.283-.698-2.686.705-1.987 1.987l.169.311c.446.82.023 1.841-.872 2.105l-.34.1c-1.4.413-1.4 2.397 0 2.81l.34.1a1.464 1.464 0 0 1 .872 2.105l-.17.31c-.698 1.283.705 2.686 1.987 1.987l.311-.169a1.464 1.464 0 0 1 2.105.872l.1.34c.413 1.4 2.397 1.4 2.81 0l.1-.34a1.464 1.464 0 0 1 2.105-.872l.31.17c1.283.698 2.686-.705 1.987-1.987l-.169-.311a1.464 1.464 0 0 1 .872-2.105l.34-.1c1.4-.413 1.4-2.397 0-2.81l-.34-.1a1.464 1.464 0 0 1-.872-2.105l.17-.31c.698-1.283-.705-2.686-1.987-1.987l-.311.169a1.464 1.464 0 0 1-2.105-.872l-.1-.34zM8 10.93a2.929 2.929 0 1 1 0-5.86 2.929 2.929 0 0 1 0 5.858z"/>
|
||||
</svg>
|
||||
System
|
||||
</h1>
|
||||
<p class="text-muted mb-0 mt-1">Server information, feature flags, and maintenance tools.</p>
|
||||
</div>
|
||||
<div class="d-none d-md-block">
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary fs-6 px-3 py-2">v{{ app_version }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row g-4 mb-4">
|
||||
<div class="col-lg-6">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M5 0a.5.5 0 0 1 .5.5V2h1V.5a.5.5 0 0 1 1 0V2h1V.5a.5.5 0 0 1 1 0V2h1V.5a.5.5 0 0 1 1 0V2A2.5 2.5 0 0 1 14 4.5h1.5a.5.5 0 0 1 0 1H14v1h1.5a.5.5 0 0 1 0 1H14v1h1.5a.5.5 0 0 1 0 1H14v1h1.5a.5.5 0 0 1 0 1H14a2.5 2.5 0 0 1-2.5 2.5v1.5a.5.5 0 0 1-1 0V14h-1v1.5a.5.5 0 0 1-1 0V14h-1v1.5a.5.5 0 0 1-1 0V14h-1v1.5a.5.5 0 0 1-1 0V14A2.5 2.5 0 0 1 2 11.5H.5a.5.5 0 0 1 0-1H2v-1H.5a.5.5 0 0 1 0-1H2v-1H.5a.5.5 0 0 1 0-1H2v-1H.5a.5.5 0 0 1 0-1H2A2.5 2.5 0 0 1 4.5 2V.5A.5.5 0 0 1 5 0zm-.5 3A1.5 1.5 0 0 0 3 4.5v7A1.5 1.5 0 0 0 4.5 13h7a1.5 1.5 0 0 0 1.5-1.5v-7A1.5 1.5 0 0 0 11.5 3h-7zM5 6.5A1.5 1.5 0 0 1 6.5 5h3A1.5 1.5 0 0 1 11 6.5v3A1.5 1.5 0 0 1 9.5 11h-3A1.5 1.5 0 0 1 5 9.5v-3zM6.5 6a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5h-3z"/>
|
||||
</svg>
|
||||
Server Information
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Runtime environment and configuration</p>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
<table class="table table-sm mb-0">
|
||||
<tbody>
|
||||
<tr><td class="text-muted" style="width:40%">Version</td><td class="fw-medium">{{ app_version }}</td></tr>
|
||||
<tr><td class="text-muted">Storage Root</td><td><code>{{ storage_root }}</code></td></tr>
|
||||
<tr><td class="text-muted">Platform</td><td>{{ platform }}</td></tr>
|
||||
<tr><td class="text-muted">Python</td><td>{{ python_version }}</td></tr>
|
||||
<tr><td class="text-muted">Rust Extension</td><td>
|
||||
{% if has_rust %}
|
||||
<span class="badge bg-success bg-opacity-10 text-success">Loaded</span>
|
||||
{% else %}
|
||||
<span class="badge bg-secondary bg-opacity-10 text-secondary">Not loaded</span>
|
||||
{% endif %}
|
||||
</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-6">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M11.5 2a1.5 1.5 0 1 0 0 3 1.5 1.5 0 0 0 0-3zM9.05 3a2.5 2.5 0 0 1 4.9 0H16v1h-2.05a2.5 2.5 0 0 1-4.9 0H0V3h9.05zM4.5 7a1.5 1.5 0 1 0 0 3 1.5 1.5 0 0 0 0-3zM2.05 8a2.5 2.5 0 0 1 4.9 0H16v1H6.95a2.5 2.5 0 0 1-4.9 0H0V8h2.05zm9.45 4a1.5 1.5 0 1 0 0 3 1.5 1.5 0 0 0 0-3zm-2.45 1a2.5 2.5 0 0 1 4.9 0H16v1h-2.05a2.5 2.5 0 0 1-4.9 0H0v-1h9.05z"/>
|
||||
</svg>
|
||||
Feature Flags
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Features configured via environment variables</p>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
<table class="table table-sm mb-0">
|
||||
<tbody>
|
||||
{% for feat in features %}
|
||||
<tr>
|
||||
<td class="text-muted" style="width:55%">{{ feat.label }}</td>
|
||||
<td class="text-end">
|
||||
{% if feat.enabled %}
|
||||
<span class="badge bg-success bg-opacity-10 text-success">Enabled</span>
|
||||
{% else %}
|
||||
<span class="badge bg-secondary bg-opacity-10 text-secondary">Disabled</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row g-4 mb-4">
|
||||
<div class="col-lg-6">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<div class="d-flex justify-content-between align-items-start">
|
||||
<div>
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M2.5 1a1 1 0 0 0-1 1v1a1 1 0 0 0 1 1H3v9a2 2 0 0 0 2 2h6a2 2 0 0 0 2-2V4h.5a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1H10a1 1 0 0 0-1-1H7a1 1 0 0 0-1 1H2.5zm3 4a.5.5 0 0 1 .5.5v7a.5.5 0 0 1-1 0v-7a.5.5 0 0 1 .5-.5zM8 5a.5.5 0 0 1 .5.5v7a.5.5 0 0 1-1 0v-7A.5.5 0 0 1 8 5zm3 .5v7a.5.5 0 0 1-1 0v-7a.5.5 0 0 1 1 0z"/>
|
||||
</svg>
|
||||
Garbage Collection
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Clean up temporary files, orphaned uploads, and stale locks</p>
|
||||
</div>
|
||||
<div>
|
||||
{% if gc_status.enabled %}
|
||||
<span class="badge bg-success bg-opacity-10 text-success">Active</span>
|
||||
{% else %}
|
||||
<span class="badge bg-secondary bg-opacity-10 text-secondary">Disabled</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
{% if gc_status.enabled %}
|
||||
<div class="d-flex gap-2 mb-3">
|
||||
<button class="btn btn-primary btn-sm d-inline-flex align-items-center" id="gcRunBtn" onclick="runGC(false)">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1 flex-shrink-0" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>
|
||||
<path d="M8 4.466V.534a.25.25 0 0 1 .41-.192l2.36 1.966c.12.1.12.284 0 .384L8.41 4.658A.25.25 0 0 1 8 4.466z"/>
|
||||
</svg>
|
||||
Run Now
|
||||
</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" id="gcDryRunBtn" onclick="runGC(true)">
|
||||
Dry Run
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div id="gcScanningBanner" class="mb-3 {% if not gc_status.scanning %}d-none{% endif %}">
|
||||
<div class="alert alert-info mb-0 small d-flex align-items-center gap-2">
|
||||
<div class="spinner-border spinner-border-sm text-info" role="status"></div>
|
||||
<span>GC in progress<span id="gcScanElapsed"></span></span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="gcResult" class="mb-3 d-none">
|
||||
<div class="alert mb-0 small" id="gcResultAlert">
|
||||
<div class="d-flex justify-content-between align-items-start">
|
||||
<div class="fw-semibold mb-1" id="gcResultTitle"></div>
|
||||
<button type="button" class="btn-close btn-close-sm" style="font-size:0.65rem" onclick="document.getElementById('gcResult').classList.add('d-none')"></button>
|
||||
</div>
|
||||
<div id="gcResultBody"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="border rounded p-3 mb-3" style="background: var(--bs-tertiary-bg, #f8f9fa);">
|
||||
<div class="d-flex align-items-center gap-2 mb-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M9.405 1.05c-.413-1.4-2.397-1.4-2.81 0l-.1.34a1.464 1.464 0 0 1-2.105.872l-.31-.17c-1.283-.698-2.686.705-1.987 1.987l.169.311c.446.82.023 1.841-.872 2.105l-.34.1c-1.4.413-1.4 2.397 0 2.81l.34.1a1.464 1.464 0 0 1 .872 2.105l-.17.31c-.698 1.283.705 2.686 1.987 1.987l.311-.169a1.464 1.464 0 0 1 2.105.872l.1.34c.413 1.4 2.397 1.4 2.81 0l.1-.34a1.464 1.464 0 0 1 2.105-.872l.31.17c1.283.698 2.686-.705 1.987-1.987l-.169-.311a1.464 1.464 0 0 1 .872-2.105l.34-.1c1.4-.413 1.4-2.397 0-2.81l-.34-.1a1.464 1.464 0 0 1-.872-2.105l.17-.31c.698-1.283-.705-2.686-1.987-1.987l-.311.169a1.464 1.464 0 0 1-2.105-.872l-.1-.34zM8 10.93a2.929 2.929 0 1 1 0-5.86 2.929 2.929 0 0 1 0 5.858z"/>
|
||||
</svg>
|
||||
<span class="small fw-semibold text-muted">Configuration</span>
|
||||
</div>
|
||||
<div class="row small">
|
||||
<div class="col-6 mb-1"><span class="text-muted">Interval:</span> {{ gc_status.interval_hours }}h</div>
|
||||
<div class="col-6 mb-1"><span class="text-muted">Dry run:</span> {{ "Yes" if gc_status.dry_run else "No" }}</div>
|
||||
<div class="col-6 mb-1"><span class="text-muted">Temp max age:</span> {{ gc_status.temp_file_max_age_hours }}h</div>
|
||||
<div class="col-6 mb-1"><span class="text-muted">Lock max age:</span> {{ gc_status.lock_file_max_age_hours }}h</div>
|
||||
<div class="col-6"><span class="text-muted">Multipart max age:</span> {{ gc_status.multipart_max_age_days }}d</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="gcHistoryContainer">
|
||||
{% if gc_history %}
|
||||
<h6 class="fw-semibold small text-muted mb-2 d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M8.515 1.019A7 7 0 0 0 8 1V0a8 8 0 0 1 .589.022l-.074.997zm2.004.45a7.003 7.003 0 0 0-.985-.299l.219-.976c.383.086.76.2 1.126.342l-.36.933zm1.37.71a7.01 7.01 0 0 0-.439-.27l.493-.87a8.025 8.025 0 0 1 .979.654l-.615.789a6.996 6.996 0 0 0-.418-.302zm1.834 1.79a6.99 6.99 0 0 0-.653-.796l.724-.69c.27.285.52.59.747.91l-.818.576zm.744 1.352a7.08 7.08 0 0 0-.214-.468l.893-.45a7.976 7.976 0 0 1 .45 1.088l-.95.313a7.023 7.023 0 0 0-.179-.483zm.53 2.507a6.991 6.991 0 0 0-.1-1.025l.985-.17c.067.386.106.778.116 1.17l-1 .025zm-.131 1.538c.033-.17.06-.339.081-.51l.993.123a7.957 7.957 0 0 1-.23 1.155l-.964-.267c.046-.165.086-.332.12-.501zm-.952 2.379c.184-.29.346-.594.486-.908l.914.405c-.16.36-.345.706-.555 1.038l-.845-.535zm-.964 1.205c.122-.122.239-.248.35-.378l.758.653a8.073 8.073 0 0 1-.401.432l-.707-.707z"/>
|
||||
<path d="M8 1a7 7 0 1 0 4.95 11.95l.707.707A8.001 8.001 0 1 1 8 0v1z"/>
|
||||
<path d="M7.5 3a.5.5 0 0 1 .5.5v5.21l3.248 1.856a.5.5 0 0 1-.496.868l-3.5-2A.5.5 0 0 1 7 8V3.5a.5.5 0 0 1 .5-.5z"/>
|
||||
</svg>
|
||||
Recent Executions
|
||||
</h6>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-sm small mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Time</th>
|
||||
<th class="text-center">Cleaned</th>
|
||||
<th class="text-center">Freed</th>
|
||||
<th class="text-center">Mode</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for exec in gc_history %}
|
||||
<tr>
|
||||
<td class="text-nowrap">{{ exec.timestamp_display }}</td>
|
||||
<td class="text-center">
|
||||
{% set r = exec.result %}
|
||||
{{ (r.temp_files_deleted|d(0)) + (r.multipart_uploads_deleted|d(0)) + (r.lock_files_deleted|d(0)) + (r.orphaned_metadata_deleted|d(0)) + (r.orphaned_versions_deleted|d(0)) + (r.empty_dirs_removed|d(0)) }}
|
||||
</td>
|
||||
<td class="text-center">{{ exec.bytes_freed_display }}</td>
|
||||
<td class="text-center">
|
||||
{% if exec.dry_run %}
|
||||
<span class="badge bg-warning bg-opacity-10 text-warning">Dry run</span>
|
||||
{% else %}
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary">Live</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="text-center py-2">
|
||||
<p class="text-muted small mb-0">No executions recorded yet.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% else %}
|
||||
<div class="text-center py-4">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="40" height="40" fill="currentColor" class="text-muted mb-2 opacity-50" viewBox="0 0 16 16">
|
||||
<path d="M2.5 1a1 1 0 0 0-1 1v1a1 1 0 0 0 1 1H3v9a2 2 0 0 0 2 2h6a2 2 0 0 0 2-2V4h.5a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1H10a1 1 0 0 0-1-1H7a1 1 0 0 0-1 1H2.5zm3 4a.5.5 0 0 1 .5.5v7a.5.5 0 0 1-1 0v-7a.5.5 0 0 1 .5-.5zM8 5a.5.5 0 0 1 .5.5v7a.5.5 0 0 1-1 0v-7A.5.5 0 0 1 8 5zm3 .5v7a.5.5 0 0 1-1 0v-7a.5.5 0 0 1 1 0z"/>
|
||||
</svg>
|
||||
<p class="text-muted mb-1">Garbage collection is not enabled.</p>
|
||||
<p class="text-muted small mb-0">Set <code>GC_ENABLED=true</code> to enable automatic cleanup.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-6">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<div class="d-flex justify-content-between align-items-start">
|
||||
<div>
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M5.338 1.59a61.44 61.44 0 0 0-2.837.856.481.481 0 0 0-.328.39c-.554 4.157.726 7.19 2.253 9.188a10.725 10.725 0 0 0 2.287 2.233c.346.244.652.42.893.533.12.057.218.095.293.118a.55.55 0 0 0 .101.025.615.615 0 0 0 .1-.025c.076-.023.174-.061.294-.118.24-.113.547-.29.893-.533a10.726 10.726 0 0 0 2.287-2.233c1.527-1.997 2.807-5.031 2.253-9.188a.48.48 0 0 0-.328-.39c-.651-.213-1.75-.56-2.837-.855C9.552 1.29 8.531 1.067 8 1.067c-.53 0-1.552.223-2.662.524zM5.072.56C6.157.265 7.31 0 8 0s1.843.265 2.928.56c1.11.3 2.229.655 2.887.87a1.54 1.54 0 0 1 1.044 1.262c.596 4.477-.787 7.795-2.465 9.99a11.775 11.775 0 0 1-2.517 2.453 7.159 7.159 0 0 1-1.048.625c-.28.132-.581.24-.829.24s-.548-.108-.829-.24a7.158 7.158 0 0 1-1.048-.625 11.777 11.777 0 0 1-2.517-2.453C1.928 10.487.545 7.169 1.141 2.692A1.54 1.54 0 0 1 2.185 1.43 62.456 62.456 0 0 1 5.072.56z"/>
|
||||
<path d="M10.854 5.146a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708 0l-1.5-1.5a.5.5 0 1 1 .708-.708L7.5 7.793l2.646-2.647a.5.5 0 0 1 .708 0z"/>
|
||||
</svg>
|
||||
Integrity Scanner
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Detect and heal corrupted objects, orphaned files, and metadata drift</p>
|
||||
</div>
|
||||
<div>
|
||||
{% if integrity_status.enabled %}
|
||||
<span class="badge bg-success bg-opacity-10 text-success">Active</span>
|
||||
{% else %}
|
||||
<span class="badge bg-secondary bg-opacity-10 text-secondary">Disabled</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
{% if integrity_status.enabled %}
|
||||
<div class="d-flex gap-2 flex-wrap mb-3">
|
||||
<button class="btn btn-primary btn-sm d-inline-flex align-items-center" id="integrityRunBtn" onclick="runIntegrity(false, false)" {% if integrity_status.scanning %}disabled{% endif %}>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1 flex-shrink-0" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>
|
||||
<path d="M8 4.466V.534a.25.25 0 0 1 .41-.192l2.36 1.966c.12.1.12.284 0 .384L8.41 4.658A.25.25 0 0 1 8 4.466z"/>
|
||||
</svg>
|
||||
Scan Now
|
||||
</button>
|
||||
<button class="btn btn-outline-warning btn-sm" id="integrityHealBtn" onclick="runIntegrity(false, true)" {% if integrity_status.scanning %}disabled{% endif %}>
|
||||
Scan & Heal
|
||||
</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" id="integrityDryRunBtn" onclick="runIntegrity(true, false)" {% if integrity_status.scanning %}disabled{% endif %}>
|
||||
Dry Run
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div id="integrityScanningBanner" class="mb-3 {% if not integrity_status.scanning %}d-none{% endif %}">
|
||||
<div class="alert alert-info mb-0 small d-flex align-items-center gap-2">
|
||||
<div class="spinner-border spinner-border-sm text-info" role="status"></div>
|
||||
<span>Scan in progress<span id="integrityScanElapsed"></span></span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="integrityResult" class="mb-3 d-none">
|
||||
<div class="alert mb-0 small" id="integrityResultAlert">
|
||||
<div class="d-flex justify-content-between align-items-start">
|
||||
<div class="fw-semibold mb-1" id="integrityResultTitle"></div>
|
||||
<button type="button" class="btn-close btn-close-sm" style="font-size:0.65rem" onclick="document.getElementById('integrityResult').classList.add('d-none')"></button>
|
||||
</div>
|
||||
<div id="integrityResultBody"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="border rounded p-3 mb-3" style="background: var(--bs-tertiary-bg, #f8f9fa);">
|
||||
<div class="d-flex align-items-center gap-2 mb-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M9.405 1.05c-.413-1.4-2.397-1.4-2.81 0l-.1.34a1.464 1.464 0 0 1-2.105.872l-.31-.17c-1.283-.698-2.686.705-1.987 1.987l.169.311c.446.82.023 1.841-.872 2.105l-.34.1c-1.4.413-1.4 2.397 0 2.81l.34.1a1.464 1.464 0 0 1 .872 2.105l-.17.31c-.698 1.283.705 2.686 1.987 1.987l.311-.169a1.464 1.464 0 0 1 2.105.872l.1.34c.413 1.4 2.397 1.4 2.81 0l.1-.34a1.464 1.464 0 0 1 2.105-.872l.31.17c1.283.698 2.686-.705 1.987-1.987l-.169-.311a1.464 1.464 0 0 1 .872-2.105l.34-.1c1.4-.413 1.4-2.397 0-2.81l-.34-.1a1.464 1.464 0 0 1-.872-2.105l.17-.31c.698-1.283-.705-2.686-1.987-1.987l-.311.169a1.464 1.464 0 0 1-2.105-.872l-.1-.34zM8 10.93a2.929 2.929 0 1 1 0-5.86 2.929 2.929 0 0 1 0 5.858z"/>
|
||||
</svg>
|
||||
<span class="small fw-semibold text-muted">Configuration</span>
|
||||
</div>
|
||||
<div class="row small">
|
||||
<div class="col-6 mb-1"><span class="text-muted">Interval:</span> {{ integrity_status.interval_hours }}h</div>
|
||||
<div class="col-6 mb-1"><span class="text-muted">Dry run:</span> {{ "Yes" if integrity_status.dry_run else "No" }}</div>
|
||||
<div class="col-6"><span class="text-muted">Batch size:</span> {{ integrity_status.batch_size }}</div>
|
||||
<div class="col-6"><span class="text-muted">Auto-heal:</span> {{ "Yes" if integrity_status.auto_heal else "No" }}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="integrityHistoryContainer">
|
||||
{% if integrity_history %}
|
||||
<h6 class="fw-semibold small text-muted mb-2 d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M8.515 1.019A7 7 0 0 0 8 1V0a8 8 0 0 1 .589.022l-.074.997zm2.004.45a7.003 7.003 0 0 0-.985-.299l.219-.976c.383.086.76.2 1.126.342l-.36.933zm1.37.71a7.01 7.01 0 0 0-.439-.27l.493-.87a8.025 8.025 0 0 1 .979.654l-.615.789a6.996 6.996 0 0 0-.418-.302zm1.834 1.79a6.99 6.99 0 0 0-.653-.796l.724-.69c.27.285.52.59.747.91l-.818.576zm.744 1.352a7.08 7.08 0 0 0-.214-.468l.893-.45a7.976 7.976 0 0 1 .45 1.088l-.95.313a7.023 7.023 0 0 0-.179-.483zm.53 2.507a6.991 6.991 0 0 0-.1-1.025l.985-.17c.067.386.106.778.116 1.17l-1 .025zm-.131 1.538c.033-.17.06-.339.081-.51l.993.123a7.957 7.957 0 0 1-.23 1.155l-.964-.267c.046-.165.086-.332.12-.501zm-.952 2.379c.184-.29.346-.594.486-.908l.914.405c-.16.36-.345.706-.555 1.038l-.845-.535zm-.964 1.205c.122-.122.239-.248.35-.378l.758.653a8.073 8.073 0 0 1-.401.432l-.707-.707z"/>
|
||||
<path d="M8 1a7 7 0 1 0 4.95 11.95l.707.707A8.001 8.001 0 1 1 8 0v1z"/>
|
||||
<path d="M7.5 3a.5.5 0 0 1 .5.5v5.21l3.248 1.856a.5.5 0 0 1-.496.868l-3.5-2A.5.5 0 0 1 7 8V3.5a.5.5 0 0 1 .5-.5z"/>
|
||||
</svg>
|
||||
Recent Scans
|
||||
</h6>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-sm small mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Time</th>
|
||||
<th class="text-center">Scanned</th>
|
||||
<th class="text-center">Issues</th>
|
||||
<th class="text-center">Healed</th>
|
||||
<th class="text-center">Mode</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for exec in integrity_history %}
|
||||
<tr>
|
||||
<td class="text-nowrap">{{ exec.timestamp_display }}</td>
|
||||
<td class="text-center">{{ exec.result.objects_scanned|d(0) }}</td>
|
||||
<td class="text-center">
|
||||
{% set total_issues = (exec.result.corrupted_objects|d(0)) + (exec.result.orphaned_objects|d(0)) + (exec.result.phantom_metadata|d(0)) + (exec.result.stale_versions|d(0)) + (exec.result.etag_cache_inconsistencies|d(0)) + (exec.result.legacy_metadata_drifts|d(0)) %}
|
||||
{% if total_issues > 0 %}
|
||||
<span class="text-danger fw-medium">{{ total_issues }}</span>
|
||||
{% else %}
|
||||
<span class="text-success">0</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="text-center">{{ exec.result.issues_healed|d(0) }}</td>
|
||||
<td class="text-center">
|
||||
{% if exec.dry_run %}
|
||||
<span class="badge bg-warning bg-opacity-10 text-warning">Dry</span>
|
||||
{% elif exec.auto_heal %}
|
||||
<span class="badge bg-success bg-opacity-10 text-success">Heal</span>
|
||||
{% else %}
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary">Scan</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="text-center py-2">
|
||||
<p class="text-muted small mb-0">No scans recorded yet.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% else %}
|
||||
<div class="text-center py-4">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="40" height="40" fill="currentColor" class="text-muted mb-2 opacity-50" viewBox="0 0 16 16">
|
||||
<path d="M5.338 1.59a61.44 61.44 0 0 0-2.837.856.481.481 0 0 0-.328.39c-.554 4.157.726 7.19 2.253 9.188a10.725 10.725 0 0 0 2.287 2.233c.346.244.652.42.893.533.12.057.218.095.293.118a.55.55 0 0 0 .101.025.615.615 0 0 0 .1-.025c.076-.023.174-.061.294-.118.24-.113.547-.29.893-.533a10.726 10.726 0 0 0 2.287-2.233c1.527-1.997 2.807-5.031 2.253-9.188a.48.48 0 0 0-.328-.39c-.651-.213-1.75-.56-2.837-.855C9.552 1.29 8.531 1.067 8 1.067c-.53 0-1.552.223-2.662.524zM5.072.56C6.157.265 7.31 0 8 0s1.843.265 2.928.56c1.11.3 2.229.655 2.887.87a1.54 1.54 0 0 1 1.044 1.262c.596 4.477-.787 7.795-2.465 9.99a11.775 11.775 0 0 1-2.517 2.453 7.159 7.159 0 0 1-1.048.625c-.28.132-.581.24-.829.24s-.548-.108-.829-.24a7.158 7.158 0 0 1-1.048-.625 11.777 11.777 0 0 1-2.517-2.453C1.928 10.487.545 7.169 1.141 2.692A1.54 1.54 0 0 1 2.185 1.43 62.456 62.456 0 0 1 5.072.56z"/>
|
||||
<path d="M10.854 5.146a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708 0l-1.5-1.5a.5.5 0 1 1 .708-.708L7.5 7.793l2.646-2.647a.5.5 0 0 1 .708 0z"/>
|
||||
</svg>
|
||||
<p class="text-muted mb-1">Integrity scanner is not enabled.</p>
|
||||
<p class="text-muted small mb-0">Set <code>INTEGRITY_ENABLED=true</code> to enable automatic scanning.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block extra_scripts %}
|
||||
<script>
|
||||
(function () {
|
||||
var csrfToken = document.querySelector('meta[name="csrf-token"]')?.getAttribute('content') || '';
|
||||
|
||||
function setLoading(btnId, loading, spinnerOnly) {
|
||||
var btn = document.getElementById(btnId);
|
||||
if (!btn) return;
|
||||
btn.disabled = loading;
|
||||
if (loading && !spinnerOnly) {
|
||||
btn.dataset.originalHtml = btn.innerHTML;
|
||||
btn.innerHTML = '<span class="spinner-border spinner-border-sm me-1" role="status"></span>Running...';
|
||||
} else if (!loading && btn.dataset.originalHtml) {
|
||||
btn.innerHTML = btn.dataset.originalHtml;
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (!bytes || bytes === 0) return '0 B';
|
||||
var units = ['B', 'KB', 'MB', 'GB'];
|
||||
var i = 0;
|
||||
var b = bytes;
|
||||
while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
|
||||
return (i === 0 ? b : b.toFixed(1)) + ' ' + units[i];
|
||||
}
|
||||
|
||||
var _displayTimezone = {{ display_timezone|tojson }};
|
||||
|
||||
function formatTimestamp(ts) {
|
||||
var d = new Date(ts * 1000);
|
||||
try {
|
||||
var opts = {year: 'numeric', month: 'short', day: '2-digit', hour: '2-digit', minute: '2-digit', hour12: false, timeZone: _displayTimezone, timeZoneName: 'short'};
|
||||
return d.toLocaleString('en-US', opts);
|
||||
} catch (e) {
|
||||
var pad = function (n) { return n < 10 ? '0' + n : '' + n; };
|
||||
return d.getUTCFullYear() + '-' + pad(d.getUTCMonth() + 1) + '-' + pad(d.getUTCDate()) +
|
||||
' ' + pad(d.getUTCHours()) + ':' + pad(d.getUTCMinutes()) + ' UTC';
|
||||
}
|
||||
}
|
||||
|
||||
var _gcHistoryIcon = '<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">' +
|
||||
'<path d="M8.515 1.019A7 7 0 0 0 8 1V0a8 8 0 0 1 .589.022l-.074.997zm2.004.45a7.003 7.003 0 0 0-.985-.299l.219-.976c.383.086.76.2 1.126.342l-.36.933zm1.37.71a7.01 7.01 0 0 0-.439-.27l.493-.87a8.025 8.025 0 0 1 .979.654l-.615.789a6.996 6.996 0 0 0-.418-.302zm1.834 1.79a6.99 6.99 0 0 0-.653-.796l.724-.69c.27.285.52.59.747.91l-.818.576zm.744 1.352a7.08 7.08 0 0 0-.214-.468l.893-.45a7.976 7.976 0 0 1 .45 1.088l-.95.313a7.023 7.023 0 0 0-.179-.483zm.53 2.507a6.991 6.991 0 0 0-.1-1.025l.985-.17c.067.386.106.778.116 1.17l-1 .025zm-.131 1.538c.033-.17.06-.339.081-.51l.993.123a7.957 7.957 0 0 1-.23 1.155l-.964-.267c.046-.165.086-.332.12-.501zm-.952 2.379c.184-.29.346-.594.486-.908l.914.405c-.16.36-.345.706-.555 1.038l-.845-.535zm-.964 1.205c.122-.122.239-.248.35-.378l.758.653a8.073 8.073 0 0 1-.401.432l-.707-.707z"/>' +
|
||||
'<path d="M8 1a7 7 0 1 0 4.95 11.95l.707.707A8.001 8.001 0 1 1 8 0v1z"/>' +
|
||||
'<path d="M7.5 3a.5.5 0 0 1 .5.5v5.21l3.248 1.856a.5.5 0 0 1-.496.868l-3.5-2A.5.5 0 0 1 7 8V3.5a.5.5 0 0 1 .5-.5z"/></svg>';
|
||||
|
||||
function _gcRefreshHistory() {
|
||||
fetch('{{ url_for("ui.system_gc_history") }}?limit=10', {
|
||||
headers: {'X-CSRFToken': csrfToken}
|
||||
})
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (hist) {
|
||||
var container = document.getElementById('gcHistoryContainer');
|
||||
if (!container) return;
|
||||
var execs = hist.executions || [];
|
||||
if (execs.length === 0) {
|
||||
container.innerHTML = '<div class="text-center py-2"><p class="text-muted small mb-0">No executions recorded yet.</p></div>';
|
||||
return;
|
||||
}
|
||||
var html = '<h6 class="fw-semibold small text-muted mb-2 d-flex align-items-center gap-2">' +
|
||||
_gcHistoryIcon + ' Recent Executions</h6>' +
|
||||
'<div class="table-responsive"><table class="table table-sm small mb-0">' +
|
||||
'<thead class="table-light"><tr><th>Time</th><th class="text-center">Cleaned</th>' +
|
||||
'<th class="text-center">Freed</th><th class="text-center">Mode</th></tr></thead><tbody>';
|
||||
execs.forEach(function (exec) {
|
||||
var r = exec.result || {};
|
||||
var cleaned = (r.temp_files_deleted || 0) + (r.multipart_uploads_deleted || 0) +
|
||||
(r.lock_files_deleted || 0) + (r.orphaned_metadata_deleted || 0) +
|
||||
(r.orphaned_versions_deleted || 0) + (r.empty_dirs_removed || 0);
|
||||
var freed = (r.temp_bytes_freed || 0) + (r.multipart_bytes_freed || 0) +
|
||||
(r.orphaned_version_bytes_freed || 0);
|
||||
var mode = exec.dry_run
|
||||
? '<span class="badge bg-warning bg-opacity-10 text-warning">Dry run</span>'
|
||||
: '<span class="badge bg-primary bg-opacity-10 text-primary">Live</span>';
|
||||
html += '<tr><td class="text-nowrap">' + formatTimestamp(exec.timestamp) + '</td>' +
|
||||
'<td class="text-center">' + cleaned + '</td>' +
|
||||
'<td class="text-center">' + formatBytes(freed) + '</td>' +
|
||||
'<td class="text-center">' + mode + '</td></tr>';
|
||||
});
|
||||
html += '</tbody></table></div>';
|
||||
container.innerHTML = html;
|
||||
})
|
||||
.catch(function () {});
|
||||
}
|
||||
|
||||
function _integrityRefreshHistory() {
|
||||
fetch('{{ url_for("ui.system_integrity_history") }}?limit=10', {
|
||||
headers: {'X-CSRFToken': csrfToken}
|
||||
})
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (hist) {
|
||||
var container = document.getElementById('integrityHistoryContainer');
|
||||
if (!container) return;
|
||||
var execs = hist.executions || [];
|
||||
if (execs.length === 0) {
|
||||
container.innerHTML = '<div class="text-center py-2"><p class="text-muted small mb-0">No scans recorded yet.</p></div>';
|
||||
return;
|
||||
}
|
||||
var html = '<h6 class="fw-semibold small text-muted mb-2 d-flex align-items-center gap-2">' +
|
||||
_gcHistoryIcon + ' Recent Scans</h6>' +
|
||||
'<div class="table-responsive"><table class="table table-sm small mb-0">' +
|
||||
'<thead class="table-light"><tr><th>Time</th><th class="text-center">Scanned</th>' +
|
||||
'<th class="text-center">Issues</th><th class="text-center">Healed</th>' +
|
||||
'<th class="text-center">Mode</th></tr></thead><tbody>';
|
||||
execs.forEach(function (exec) {
|
||||
var r = exec.result || {};
|
||||
var issues = (r.corrupted_objects || 0) + (r.orphaned_objects || 0) +
|
||||
(r.phantom_metadata || 0) + (r.stale_versions || 0) +
|
||||
(r.etag_cache_inconsistencies || 0) + (r.legacy_metadata_drifts || 0);
|
||||
var issueHtml = issues > 0
|
||||
? '<span class="text-danger fw-medium">' + issues + '</span>'
|
||||
: '<span class="text-success">0</span>';
|
||||
var mode = exec.dry_run
|
||||
? '<span class="badge bg-warning bg-opacity-10 text-warning">Dry</span>'
|
||||
: (exec.auto_heal
|
||||
? '<span class="badge bg-success bg-opacity-10 text-success">Heal</span>'
|
||||
: '<span class="badge bg-primary bg-opacity-10 text-primary">Scan</span>');
|
||||
html += '<tr><td class="text-nowrap">' + formatTimestamp(exec.timestamp) + '</td>' +
|
||||
'<td class="text-center">' + (r.objects_scanned || 0) + '</td>' +
|
||||
'<td class="text-center">' + issueHtml + '</td>' +
|
||||
'<td class="text-center">' + (r.issues_healed || 0) + '</td>' +
|
||||
'<td class="text-center">' + mode + '</td></tr>';
|
||||
});
|
||||
html += '</tbody></table></div>';
|
||||
container.innerHTML = html;
|
||||
})
|
||||
.catch(function () {});
|
||||
}
|
||||
|
||||
var _gcPollTimer = null;
|
||||
var _gcLastDryRun = false;
|
||||
|
||||
function _gcSetScanning(scanning) {
|
||||
var banner = document.getElementById('gcScanningBanner');
|
||||
var btns = ['gcRunBtn', 'gcDryRunBtn'];
|
||||
if (scanning) {
|
||||
banner.classList.remove('d-none');
|
||||
btns.forEach(function (id) {
|
||||
var el = document.getElementById(id);
|
||||
if (el) el.disabled = true;
|
||||
});
|
||||
} else {
|
||||
banner.classList.add('d-none');
|
||||
document.getElementById('gcScanElapsed').textContent = '';
|
||||
btns.forEach(function (id) {
|
||||
var el = document.getElementById(id);
|
||||
if (el) el.disabled = false;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function _gcShowResult(data, dryRun) {
|
||||
var container = document.getElementById('gcResult');
|
||||
var alert = document.getElementById('gcResultAlert');
|
||||
var title = document.getElementById('gcResultTitle');
|
||||
var body = document.getElementById('gcResultBody');
|
||||
container.classList.remove('d-none');
|
||||
|
||||
var totalItems = (data.temp_files_deleted || 0) + (data.multipart_uploads_deleted || 0) +
|
||||
(data.lock_files_deleted || 0) + (data.orphaned_metadata_deleted || 0) +
|
||||
(data.orphaned_versions_deleted || 0) + (data.empty_dirs_removed || 0);
|
||||
var totalFreed = (data.temp_bytes_freed || 0) + (data.multipart_bytes_freed || 0) +
|
||||
(data.orphaned_version_bytes_freed || 0);
|
||||
|
||||
alert.className = totalItems > 0 ? 'alert alert-success mb-0 small' : 'alert alert-info mb-0 small';
|
||||
title.textContent = (dryRun ? '[Dry Run] ' : '') + 'Completed in ' + (data.execution_time_seconds || 0).toFixed(2) + 's';
|
||||
|
||||
var lines = [];
|
||||
if (data.temp_files_deleted) lines.push('Temp files: ' + data.temp_files_deleted + ' (' + formatBytes(data.temp_bytes_freed) + ')');
|
||||
if (data.multipart_uploads_deleted) lines.push('Multipart uploads: ' + data.multipart_uploads_deleted + ' (' + formatBytes(data.multipart_bytes_freed) + ')');
|
||||
if (data.lock_files_deleted) lines.push('Lock files: ' + data.lock_files_deleted);
|
||||
if (data.orphaned_metadata_deleted) lines.push('Orphaned metadata: ' + data.orphaned_metadata_deleted);
|
||||
if (data.orphaned_versions_deleted) lines.push('Orphaned versions: ' + data.orphaned_versions_deleted + ' (' + formatBytes(data.orphaned_version_bytes_freed) + ')');
|
||||
if (data.empty_dirs_removed) lines.push('Empty directories: ' + data.empty_dirs_removed);
|
||||
if (totalItems === 0) lines.push('Nothing to clean up.');
|
||||
if (totalFreed > 0) lines.push('Total freed: ' + formatBytes(totalFreed));
|
||||
if (data.errors && data.errors.length > 0) lines.push('Errors: ' + data.errors.join(', '));
|
||||
|
||||
body.innerHTML = lines.join('<br>');
|
||||
}
|
||||
|
||||
function _gcPoll() {
|
||||
fetch('{{ url_for("ui.system_gc_status") }}', {
|
||||
headers: {'X-CSRFToken': csrfToken}
|
||||
})
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (status) {
|
||||
if (status.scanning) {
|
||||
var elapsed = status.scan_elapsed_seconds || 0;
|
||||
document.getElementById('gcScanElapsed').textContent = ' (' + elapsed.toFixed(0) + 's)';
|
||||
_gcPollTimer = setTimeout(_gcPoll, 2000);
|
||||
} else {
|
||||
_gcSetScanning(false);
|
||||
_gcRefreshHistory();
|
||||
fetch('{{ url_for("ui.system_gc_history") }}?limit=1', {
|
||||
headers: {'X-CSRFToken': csrfToken}
|
||||
})
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (hist) {
|
||||
if (hist.executions && hist.executions.length > 0) {
|
||||
var latest = hist.executions[0];
|
||||
_gcShowResult(latest.result, latest.dry_run);
|
||||
}
|
||||
})
|
||||
.catch(function () {});
|
||||
}
|
||||
})
|
||||
.catch(function () {
|
||||
_gcPollTimer = setTimeout(_gcPoll, 3000);
|
||||
});
|
||||
}
|
||||
|
||||
window.runGC = function (dryRun) {
|
||||
_gcLastDryRun = dryRun;
|
||||
document.getElementById('gcResult').classList.add('d-none');
|
||||
_gcSetScanning(true);
|
||||
|
||||
fetch('{{ url_for("ui.system_gc_run") }}', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json', 'X-CSRFToken': csrfToken},
|
||||
body: JSON.stringify({dry_run: dryRun})
|
||||
})
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (data) {
|
||||
if (data.error) {
|
||||
_gcSetScanning(false);
|
||||
var container = document.getElementById('gcResult');
|
||||
var alert = document.getElementById('gcResultAlert');
|
||||
var title = document.getElementById('gcResultTitle');
|
||||
var body = document.getElementById('gcResultBody');
|
||||
container.classList.remove('d-none');
|
||||
alert.className = 'alert alert-danger mb-0 small';
|
||||
title.textContent = 'Error';
|
||||
body.textContent = data.error;
|
||||
return;
|
||||
}
|
||||
_gcPollTimer = setTimeout(_gcPoll, 2000);
|
||||
})
|
||||
.catch(function (err) {
|
||||
_gcSetScanning(false);
|
||||
var container = document.getElementById('gcResult');
|
||||
var alert = document.getElementById('gcResultAlert');
|
||||
var title = document.getElementById('gcResultTitle');
|
||||
var body = document.getElementById('gcResultBody');
|
||||
container.classList.remove('d-none');
|
||||
alert.className = 'alert alert-danger mb-0 small';
|
||||
title.textContent = 'Error';
|
||||
body.textContent = err.message;
|
||||
});
|
||||
};
|
||||
|
||||
{% if gc_status.scanning %}
|
||||
_gcSetScanning(true);
|
||||
_gcPollTimer = setTimeout(_gcPoll, 2000);
|
||||
{% endif %}
|
||||
|
||||
var _integrityPollTimer = null;
|
||||
var _integrityLastMode = {dryRun: false, autoHeal: false};
|
||||
|
||||
function _integritySetScanning(scanning) {
|
||||
var banner = document.getElementById('integrityScanningBanner');
|
||||
var btns = ['integrityRunBtn', 'integrityHealBtn', 'integrityDryRunBtn'];
|
||||
if (scanning) {
|
||||
banner.classList.remove('d-none');
|
||||
btns.forEach(function (id) {
|
||||
var el = document.getElementById(id);
|
||||
if (el) el.disabled = true;
|
||||
});
|
||||
} else {
|
||||
banner.classList.add('d-none');
|
||||
document.getElementById('integrityScanElapsed').textContent = '';
|
||||
btns.forEach(function (id) {
|
||||
var el = document.getElementById(id);
|
||||
if (el) el.disabled = false;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function _integrityShowResult(data, dryRun, autoHeal) {
|
||||
var container = document.getElementById('integrityResult');
|
||||
var alert = document.getElementById('integrityResultAlert');
|
||||
var title = document.getElementById('integrityResultTitle');
|
||||
var body = document.getElementById('integrityResultBody');
|
||||
container.classList.remove('d-none');
|
||||
|
||||
var totalIssues = (data.corrupted_objects || 0) + (data.orphaned_objects || 0) +
|
||||
(data.phantom_metadata || 0) + (data.stale_versions || 0) +
|
||||
(data.etag_cache_inconsistencies || 0) + (data.legacy_metadata_drifts || 0);
|
||||
|
||||
var prefix = dryRun ? '[Dry Run] ' : (autoHeal ? '[Heal] ' : '');
|
||||
alert.className = totalIssues > 0 ? 'alert alert-warning mb-0 small' : 'alert alert-success mb-0 small';
|
||||
title.textContent = prefix + 'Completed in ' + (data.execution_time_seconds || 0).toFixed(2) + 's';
|
||||
|
||||
var lines = [];
|
||||
lines.push('Scanned: ' + (data.objects_scanned || 0) + ' objects in ' + (data.buckets_scanned || 0) + ' buckets');
|
||||
if (totalIssues === 0) {
|
||||
lines.push('No issues found.');
|
||||
} else {
|
||||
if (data.corrupted_objects) lines.push('Corrupted objects: ' + data.corrupted_objects);
|
||||
if (data.orphaned_objects) lines.push('Orphaned objects: ' + data.orphaned_objects);
|
||||
if (data.phantom_metadata) lines.push('Phantom metadata: ' + data.phantom_metadata);
|
||||
if (data.stale_versions) lines.push('Stale versions: ' + data.stale_versions);
|
||||
if (data.etag_cache_inconsistencies) lines.push('ETag inconsistencies: ' + data.etag_cache_inconsistencies);
|
||||
if (data.legacy_metadata_drifts) lines.push('Legacy metadata drifts: ' + data.legacy_metadata_drifts);
|
||||
if (data.issues_healed) lines.push('Issues healed: ' + data.issues_healed);
|
||||
}
|
||||
if (data.errors && data.errors.length > 0) lines.push('Errors: ' + data.errors.join(', '));
|
||||
|
||||
body.innerHTML = lines.join('<br>');
|
||||
}
|
||||
|
||||
function _integrityPoll() {
|
||||
fetch('{{ url_for("ui.system_integrity_status") }}', {
|
||||
headers: {'X-CSRFToken': csrfToken}
|
||||
})
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (status) {
|
||||
if (status.scanning) {
|
||||
var elapsed = status.scan_elapsed_seconds || 0;
|
||||
document.getElementById('integrityScanElapsed').textContent = ' (' + elapsed.toFixed(0) + 's)';
|
||||
_integrityPollTimer = setTimeout(_integrityPoll, 2000);
|
||||
} else {
|
||||
_integritySetScanning(false);
|
||||
_integrityRefreshHistory();
|
||||
fetch('{{ url_for("ui.system_integrity_history") }}?limit=1', {
|
||||
headers: {'X-CSRFToken': csrfToken}
|
||||
})
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (hist) {
|
||||
if (hist.executions && hist.executions.length > 0) {
|
||||
var latest = hist.executions[0];
|
||||
_integrityShowResult(latest.result, latest.dry_run, latest.auto_heal);
|
||||
}
|
||||
})
|
||||
.catch(function () {});
|
||||
}
|
||||
})
|
||||
.catch(function () {
|
||||
_integrityPollTimer = setTimeout(_integrityPoll, 3000);
|
||||
});
|
||||
}
|
||||
|
||||
window.runIntegrity = function (dryRun, autoHeal) {
|
||||
_integrityLastMode = {dryRun: dryRun, autoHeal: autoHeal};
|
||||
document.getElementById('integrityResult').classList.add('d-none');
|
||||
_integritySetScanning(true);
|
||||
|
||||
fetch('{{ url_for("ui.system_integrity_run") }}', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json', 'X-CSRFToken': csrfToken},
|
||||
body: JSON.stringify({dry_run: dryRun, auto_heal: autoHeal})
|
||||
})
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (data) {
|
||||
if (data.error) {
|
||||
_integritySetScanning(false);
|
||||
var container = document.getElementById('integrityResult');
|
||||
var alert = document.getElementById('integrityResultAlert');
|
||||
var title = document.getElementById('integrityResultTitle');
|
||||
var body = document.getElementById('integrityResultBody');
|
||||
container.classList.remove('d-none');
|
||||
alert.className = 'alert alert-danger mb-0 small';
|
||||
title.textContent = 'Error';
|
||||
body.textContent = data.error;
|
||||
return;
|
||||
}
|
||||
_integrityPollTimer = setTimeout(_integrityPoll, 2000);
|
||||
})
|
||||
.catch(function (err) {
|
||||
_integritySetScanning(false);
|
||||
var container = document.getElementById('integrityResult');
|
||||
var alert = document.getElementById('integrityResultAlert');
|
||||
var title = document.getElementById('integrityResultTitle');
|
||||
var body = document.getElementById('integrityResultBody');
|
||||
container.classList.remove('d-none');
|
||||
alert.className = 'alert alert-danger mb-0 small';
|
||||
title.textContent = 'Error';
|
||||
body.textContent = err.message;
|
||||
});
|
||||
};
|
||||
|
||||
{% if integrity_status.scanning %}
|
||||
_integritySetScanning(true);
|
||||
_integrityPollTimer = setTimeout(_integrityPoll, 2000);
|
||||
{% endif %}
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
367
templates/website_domains.html
Normal file
367
templates/website_domains.html
Normal file
@@ -0,0 +1,367 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Website Domains - MyFSIO Console{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="page-header d-flex justify-content-between align-items-center mb-4">
|
||||
<div>
|
||||
<p class="text-uppercase text-muted small mb-1">Website Hosting</p>
|
||||
<h1 class="h3 mb-1 d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
Domain Mappings
|
||||
</h1>
|
||||
<p class="text-muted mb-0 mt-1">Map custom domains to buckets for static website hosting.</p>
|
||||
</div>
|
||||
<div class="d-none d-md-block">
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary fs-6 px-3 py-2">
|
||||
{{ mappings|length }} mapping{{ 's' if mappings|length != 1 else '' }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row g-4">
|
||||
<div class="col-lg-4 col-md-5">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 2a.5.5 0 0 1 .5.5v5h5a.5.5 0 0 1 0 1h-5v5a.5.5 0 0 1-1 0v-5h-5a.5.5 0 0 1 0-1h5v-5A.5.5 0 0 1 8 2Z"/>
|
||||
</svg>
|
||||
Add Domain Mapping
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Point a custom domain to a bucket</p>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
<form method="POST" action="{{ url_for('ui.create_website_domain') }}" id="createDomainForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="mb-3">
|
||||
<label for="domain" class="form-label fw-medium">Domain</label>
|
||||
<input type="text" class="form-control" id="domain" name="domain" required
|
||||
placeholder="www.example.com"
|
||||
pattern="^[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?)*$"
|
||||
title="Enter a valid hostname (e.g. www.example.com). Do not include http:// or trailing slashes.">
|
||||
<div class="form-text">Hostname only — no <code>http://</code> prefix or trailing slash.</div>
|
||||
<div class="invalid-feedback">Enter a valid hostname like www.example.com</div>
|
||||
</div>
|
||||
<div id="domainPreview" class="alert alert-light border small py-2 px-3 mb-3 d-none">
|
||||
<span class="text-muted">Will be accessible at:</span>
|
||||
<code id="domainPreviewUrl" class="ms-1"></code>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="bucket" class="form-label fw-medium">Bucket</label>
|
||||
{% if buckets %}
|
||||
<select class="form-select" id="bucket" name="bucket" required>
|
||||
<option value="" selected disabled>Select a bucket</option>
|
||||
{% for b in buckets %}
|
||||
<option value="{{ b }}">{{ b }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% else %}
|
||||
<input type="text" class="form-control" id="bucket" name="bucket" required placeholder="my-site-bucket">
|
||||
{% endif %}
|
||||
<div class="form-text">The bucket must have website hosting enabled.</div>
|
||||
</div>
|
||||
<div class="d-grid">
|
||||
<button type="submit" class="btn btn-primary" id="addMappingBtn">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 2a.5.5 0 0 1 .5.5v5h5a.5.5 0 0 1 0 1h-5v5a.5.5 0 0 1-1 0v-5h-5a.5.5 0 0 1 0-1h5v-5A.5.5 0 0 1 8 2Z"/>
|
||||
</svg>
|
||||
Add Mapping
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card shadow-sm border-0 mt-4" style="border-radius: 1rem;">
|
||||
<div class="card-body px-4 py-3">
|
||||
<h6 class="fw-semibold mb-2 d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
|
||||
</svg>
|
||||
How it works
|
||||
</h6>
|
||||
<ol class="small text-muted mb-0 ps-3">
|
||||
<li class="mb-1">Enable website hosting on a bucket (Properties tab)</li>
|
||||
<li class="mb-1">Create a domain mapping here</li>
|
||||
<li>Point your DNS (A/CNAME) to this server</li>
|
||||
</ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-8 col-md-7">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<div class="d-flex justify-content-between align-items-center mb-1">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-0">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M4.715 6.542 3.343 7.914a3 3 0 1 0 4.243 4.243l1.828-1.829A3 3 0 0 0 8.586 5.5L8 6.086a1.002 1.002 0 0 0-.154.199 2 2 0 0 1 .861 3.337L6.88 11.45a2 2 0 1 1-2.83-2.83l.793-.792a4.018 4.018 0 0 1-.128-1.287z"/>
|
||||
<path d="M6.586 4.672A3 3 0 0 0 7.414 9.5l.775-.776a2 2 0 0 1-.896-3.346L9.12 3.55a2 2 0 1 1 2.83 2.83l-.793.792c.112.42.155.855.128 1.287l1.372-1.372a3 3 0 1 0-4.243-4.243L6.586 4.672z"/>
|
||||
</svg>
|
||||
Active Mappings
|
||||
</h5>
|
||||
</div>
|
||||
<p class="text-muted small mb-0">Domains currently serving website content</p>
|
||||
{% if mappings|length > 3 %}
|
||||
<div class="mt-3">
|
||||
<div class="search-input-wrapper">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="search-icon" viewBox="0 0 16 16">
|
||||
<path d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0z"/>
|
||||
</svg>
|
||||
<input type="text" class="form-control" id="domainSearch" placeholder="Filter by domain or bucket..." autocomplete="off" />
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
{% if mappings %}
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover align-middle mb-0" id="domainTable">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th scope="col">Domain</th>
|
||||
<th scope="col">Bucket</th>
|
||||
<th scope="col" class="text-end">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for m in mappings %}
|
||||
<tr data-domain="{{ m.domain }}" data-bucket="{{ m.bucket }}">
|
||||
<td>
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-success flex-shrink-0" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
<div>
|
||||
<code class="fw-medium">{{ m.domain }}</code>
|
||||
<div class="text-muted small">http://{{ m.domain }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
<td><span class="badge bg-primary bg-opacity-10 text-primary">{{ m.bucket }}</span></td>
|
||||
<td class="text-end">
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
<button type="button" class="btn btn-outline-secondary"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#editDomainModal"
|
||||
data-domain="{{ m.domain }}"
|
||||
data-bucket="{{ m.bucket }}"
|
||||
title="Edit mapping">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-danger"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#deleteDomainModal"
|
||||
data-domain="{{ m.domain }}"
|
||||
data-bucket="{{ m.bucket }}"
|
||||
title="Delete mapping">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div id="noSearchResults" class="text-center py-4 d-none">
|
||||
<p class="text-muted mb-0">No mappings match your search.</p>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="empty-state text-center py-5">
|
||||
<div class="empty-state-icon mx-auto mb-3">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
</div>
|
||||
<h5 class="fw-semibold mb-2">No domain mappings yet</h5>
|
||||
<p class="text-muted mb-0">Add your first domain mapping to serve a bucket as a static website.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="editDomainModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header border-0 pb-0">
|
||||
<h5 class="modal-title fw-semibold">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5zm-9.761 5.175-.106.106-1.528 3.821 3.821-1.528.106-.106A.5.5 0 0 1 5 12.5V12h-.5a.5.5 0 0 1-.5-.5V11h-.5a.5.5 0 0 1-.468-.325z"/>
|
||||
</svg>
|
||||
Edit Domain Mapping
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form method="POST" id="editDomainForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="modal-body">
|
||||
<div class="mb-3">
|
||||
<label class="form-label fw-medium">Domain</label>
|
||||
<input type="text" class="form-control bg-light" id="editDomainName" disabled>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="editBucket" class="form-label fw-medium">Bucket</label>
|
||||
{% if buckets %}
|
||||
<select class="form-select" id="editBucket" name="bucket" required>
|
||||
{% for b in buckets %}
|
||||
<option value="{{ b }}">{{ b }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% else %}
|
||||
<input type="text" class="form-control" id="editBucket" name="bucket" required>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
|
||||
</svg>
|
||||
Save
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="deleteDomainModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<form method="POST" id="deleteDomainForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="modal-header border-0 pb-0">
|
||||
<h5 class="modal-title fw-semibold">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-danger" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Delete Domain Mapping
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Are you sure you want to delete the mapping for <strong><code id="deleteDomainName"></code></strong>?</p>
|
||||
<p class="text-muted small mb-0">Mapped to bucket: <code id="deleteBucketName"></code></p>
|
||||
<div class="alert alert-warning d-flex align-items-start small mt-3 mb-0" role="alert">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="flex-shrink-0 me-2 mt-0" viewBox="0 0 16 16">
|
||||
<path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/>
|
||||
</svg>
|
||||
<div>This domain will stop serving website content immediately.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-danger">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block extra_scripts %}
|
||||
<script>
|
||||
(function () {
|
||||
function normalizeDomain(val) {
|
||||
val = val.trim().toLowerCase();
|
||||
if (val.indexOf('https://') === 0) val = val.substring(8);
|
||||
else if (val.indexOf('http://') === 0) val = val.substring(7);
|
||||
var slashIdx = val.indexOf('/');
|
||||
if (slashIdx !== -1) val = val.substring(0, slashIdx);
|
||||
var qIdx = val.indexOf('?');
|
||||
if (qIdx !== -1) val = val.substring(0, qIdx);
|
||||
var hIdx = val.indexOf('#');
|
||||
if (hIdx !== -1) val = val.substring(0, hIdx);
|
||||
var colonIdx = val.indexOf(':');
|
||||
if (colonIdx !== -1) val = val.substring(0, colonIdx);
|
||||
return val;
|
||||
}
|
||||
|
||||
var domainInput = document.getElementById('domain');
|
||||
var preview = document.getElementById('domainPreview');
|
||||
var previewUrl = document.getElementById('domainPreviewUrl');
|
||||
if (domainInput && preview) {
|
||||
domainInput.addEventListener('input', function () {
|
||||
var clean = normalizeDomain(this.value);
|
||||
if (clean && /^[a-z0-9]([a-z0-9\-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9\-]*[a-z0-9])?)*$/.test(clean)) {
|
||||
previewUrl.textContent = 'http://' + clean;
|
||||
preview.classList.remove('d-none');
|
||||
} else {
|
||||
preview.classList.add('d-none');
|
||||
}
|
||||
});
|
||||
|
||||
var createForm = document.getElementById('createDomainForm');
|
||||
if (createForm) {
|
||||
createForm.addEventListener('submit', function () {
|
||||
domainInput.value = normalizeDomain(domainInput.value);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var editModal = document.getElementById('editDomainModal');
|
||||
if (editModal) {
|
||||
editModal.addEventListener('show.bs.modal', function (event) {
|
||||
var btn = event.relatedTarget;
|
||||
var domain = btn.getAttribute('data-domain');
|
||||
var bucket = btn.getAttribute('data-bucket');
|
||||
document.getElementById('editDomainName').value = domain;
|
||||
var editBucket = document.getElementById('editBucket');
|
||||
editBucket.value = bucket;
|
||||
document.getElementById('editDomainForm').action = '{{ url_for("ui.update_website_domain", domain="__DOMAIN__") }}'.replace('__DOMAIN__', encodeURIComponent(domain));
|
||||
});
|
||||
}
|
||||
|
||||
var deleteModal = document.getElementById('deleteDomainModal');
|
||||
if (deleteModal) {
|
||||
deleteModal.addEventListener('show.bs.modal', function (event) {
|
||||
var btn = event.relatedTarget;
|
||||
var domain = btn.getAttribute('data-domain');
|
||||
var bucket = btn.getAttribute('data-bucket') || '';
|
||||
document.getElementById('deleteDomainName').textContent = domain;
|
||||
document.getElementById('deleteBucketName').textContent = bucket;
|
||||
document.getElementById('deleteDomainForm').action = '{{ url_for("ui.delete_website_domain", domain="__DOMAIN__") }}'.replace('__DOMAIN__', encodeURIComponent(domain));
|
||||
});
|
||||
}
|
||||
|
||||
var searchInput = document.getElementById('domainSearch');
|
||||
if (searchInput) {
|
||||
searchInput.addEventListener('input', function () {
|
||||
var q = this.value.toLowerCase();
|
||||
var rows = document.querySelectorAll('#domainTable tbody tr');
|
||||
var visible = 0;
|
||||
rows.forEach(function (row) {
|
||||
var domain = (row.getAttribute('data-domain') || '').toLowerCase();
|
||||
var bucket = (row.getAttribute('data-bucket') || '').toLowerCase();
|
||||
var match = !q || domain.indexOf(q) !== -1 || bucket.indexOf(q) !== -1;
|
||||
row.style.display = match ? '' : 'none';
|
||||
if (match) visible++;
|
||||
});
|
||||
var noResults = document.getElementById('noSearchResults');
|
||||
if (noResults) {
|
||||
noResults.classList.toggle('d-none', visible > 0);
|
||||
}
|
||||
});
|
||||
}
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -27,7 +27,10 @@ def app(tmp_path: Path):
|
||||
"access_key": "test",
|
||||
"secret_key": "secret",
|
||||
"display_name": "Test User",
|
||||
"policies": [{"bucket": "*", "actions": ["list", "read", "write", "delete", "policy"]}],
|
||||
"policies": [{"bucket": "*", "actions": ["list", "read", "write", "delete", "policy",
|
||||
"create_bucket", "delete_bucket", "share", "versioning", "tagging",
|
||||
"encryption", "cors", "lifecycle", "replication", "quota",
|
||||
"object_lock", "notification", "logging", "website"]}],
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -43,6 +46,11 @@ def app(tmp_path: Path):
|
||||
}
|
||||
)
|
||||
yield flask_app
|
||||
storage = flask_app.extensions.get("object_storage")
|
||||
if storage:
|
||||
base = getattr(storage, "storage", storage)
|
||||
if hasattr(base, "shutdown_stats"):
|
||||
base.shutdown_stats()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
|
||||
156
tests/test_conditional_headers.py
Normal file
156
tests/test_conditional_headers.py
Normal file
@@ -0,0 +1,156 @@
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def bucket(client, signer):
|
||||
headers = signer("PUT", "/cond-test")
|
||||
client.put("/cond-test", headers=headers)
|
||||
return "cond-test"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def uploaded(client, signer, bucket):
|
||||
body = b"hello conditional"
|
||||
etag = hashlib.md5(body).hexdigest()
|
||||
headers = signer("PUT", f"/{bucket}/obj.txt", body=body)
|
||||
resp = client.put(f"/{bucket}/obj.txt", headers=headers, data=body)
|
||||
last_modified = resp.headers.get("Last-Modified")
|
||||
return {"etag": etag, "last_modified": last_modified}
|
||||
|
||||
|
||||
class TestIfMatch:
|
||||
def test_get_matching_etag(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-Match": f'"{uploaded["etag"]}"'})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_get_non_matching_etag(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-Match": '"wrongetag"'})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 412
|
||||
|
||||
def test_head_matching_etag(self, client, signer, bucket, uploaded):
|
||||
headers = signer("HEAD", f"/{bucket}/obj.txt", headers={"If-Match": f'"{uploaded["etag"]}"'})
|
||||
resp = client.head(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_head_non_matching_etag(self, client, signer, bucket, uploaded):
|
||||
headers = signer("HEAD", f"/{bucket}/obj.txt", headers={"If-Match": '"wrongetag"'})
|
||||
resp = client.head(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 412
|
||||
|
||||
def test_wildcard_match(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-Match": "*"})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_multiple_etags_one_matches(self, client, signer, bucket, uploaded):
|
||||
etag_list = f'"bad1", "{uploaded["etag"]}", "bad2"'
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-Match": etag_list})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_multiple_etags_none_match(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-Match": '"bad1", "bad2"'})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 412
|
||||
|
||||
|
||||
class TestIfNoneMatch:
|
||||
def test_get_matching_etag_returns_304(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-None-Match": f'"{uploaded["etag"]}"'})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 304
|
||||
assert uploaded["etag"] in resp.headers.get("ETag", "")
|
||||
|
||||
def test_get_non_matching_etag_returns_200(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-None-Match": '"wrongetag"'})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_head_matching_etag_returns_304(self, client, signer, bucket, uploaded):
|
||||
headers = signer("HEAD", f"/{bucket}/obj.txt", headers={"If-None-Match": f'"{uploaded["etag"]}"'})
|
||||
resp = client.head(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 304
|
||||
|
||||
def test_head_non_matching_etag_returns_200(self, client, signer, bucket, uploaded):
|
||||
headers = signer("HEAD", f"/{bucket}/obj.txt", headers={"If-None-Match": '"wrongetag"'})
|
||||
resp = client.head(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_wildcard_returns_304(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-None-Match": "*"})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 304
|
||||
|
||||
|
||||
class TestIfModifiedSince:
|
||||
def test_not_modified_returns_304(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-Modified-Since": "Sun, 01 Jan 2034 00:00:00 GMT"})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 304
|
||||
assert "ETag" in resp.headers
|
||||
|
||||
def test_modified_returns_200(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-Modified-Since": "Sun, 01 Jan 2000 00:00:00 GMT"})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_head_not_modified(self, client, signer, bucket, uploaded):
|
||||
headers = signer("HEAD", f"/{bucket}/obj.txt", headers={"If-Modified-Since": "Sun, 01 Jan 2034 00:00:00 GMT"})
|
||||
resp = client.head(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 304
|
||||
|
||||
def test_if_none_match_takes_precedence(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={
|
||||
"If-None-Match": '"wrongetag"',
|
||||
"If-Modified-Since": "Sun, 01 Jan 2034 00:00:00 GMT",
|
||||
})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
class TestIfUnmodifiedSince:
|
||||
def test_unmodified_returns_200(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-Unmodified-Since": "Sun, 01 Jan 2034 00:00:00 GMT"})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_modified_returns_412(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={"If-Unmodified-Since": "Sun, 01 Jan 2000 00:00:00 GMT"})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 412
|
||||
|
||||
def test_head_modified_returns_412(self, client, signer, bucket, uploaded):
|
||||
headers = signer("HEAD", f"/{bucket}/obj.txt", headers={"If-Unmodified-Since": "Sun, 01 Jan 2000 00:00:00 GMT"})
|
||||
resp = client.head(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 412
|
||||
|
||||
def test_if_match_takes_precedence(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={
|
||||
"If-Match": f'"{uploaded["etag"]}"',
|
||||
"If-Unmodified-Since": "Sun, 01 Jan 2000 00:00:00 GMT",
|
||||
})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
class TestConditionalWithRange:
|
||||
def test_if_match_with_range(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={
|
||||
"If-Match": f'"{uploaded["etag"]}"',
|
||||
"Range": "bytes=0-4",
|
||||
})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 206
|
||||
|
||||
def test_if_match_fails_with_range(self, client, signer, bucket, uploaded):
|
||||
headers = signer("GET", f"/{bucket}/obj.txt", headers={
|
||||
"If-Match": '"wrongetag"',
|
||||
"Range": "bytes=0-4",
|
||||
})
|
||||
resp = client.get(f"/{bucket}/obj.txt", headers=headers)
|
||||
assert resp.status_code == 412
|
||||
@@ -53,7 +53,9 @@ def test_special_characters_in_metadata(tmp_path: Path):
|
||||
assert meta["special"] == "!@#$%^&*()"
|
||||
|
||||
def test_disk_full_scenario(tmp_path: Path, monkeypatch):
|
||||
# Simulate disk full by mocking write to fail
|
||||
import app.storage as _storage_mod
|
||||
monkeypatch.setattr(_storage_mod, "_HAS_RUST", False)
|
||||
|
||||
storage = ObjectStorage(tmp_path)
|
||||
storage.create_bucket("full")
|
||||
|
||||
|
||||
356
tests/test_gc.py
Normal file
356
tests/test_gc.py
Normal file
@@ -0,0 +1,356 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gc import GarbageCollector, GCResult
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage_root(tmp_path):
|
||||
root = tmp_path / "data"
|
||||
root.mkdir()
|
||||
sys_root = root / ".myfsio.sys"
|
||||
sys_root.mkdir()
|
||||
(sys_root / "config").mkdir(parents=True)
|
||||
(sys_root / "tmp").mkdir()
|
||||
(sys_root / "multipart").mkdir()
|
||||
(sys_root / "buckets").mkdir()
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gc(storage_root):
|
||||
return GarbageCollector(
|
||||
storage_root=storage_root,
|
||||
interval_hours=1.0,
|
||||
temp_file_max_age_hours=1.0,
|
||||
multipart_max_age_days=1,
|
||||
lock_file_max_age_hours=0.5,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
|
||||
def _make_old(path, hours=48):
|
||||
old_time = time.time() - hours * 3600
|
||||
os.utime(path, (old_time, old_time))
|
||||
|
||||
|
||||
class TestTempFileCleanup:
|
||||
def test_old_temp_files_deleted(self, storage_root, gc):
|
||||
tmp_dir = storage_root / ".myfsio.sys" / "tmp"
|
||||
old_file = tmp_dir / "abc123.tmp"
|
||||
old_file.write_bytes(b"x" * 1000)
|
||||
_make_old(old_file, hours=48)
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.temp_files_deleted == 1
|
||||
assert result.temp_bytes_freed == 1000
|
||||
assert not old_file.exists()
|
||||
|
||||
def test_recent_temp_files_kept(self, storage_root, gc):
|
||||
tmp_dir = storage_root / ".myfsio.sys" / "tmp"
|
||||
new_file = tmp_dir / "recent.tmp"
|
||||
new_file.write_bytes(b"data")
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.temp_files_deleted == 0
|
||||
assert new_file.exists()
|
||||
|
||||
def test_dry_run_keeps_files(self, storage_root, gc):
|
||||
gc.dry_run = True
|
||||
tmp_dir = storage_root / ".myfsio.sys" / "tmp"
|
||||
old_file = tmp_dir / "stale.tmp"
|
||||
old_file.write_bytes(b"x" * 500)
|
||||
_make_old(old_file, hours=48)
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.temp_files_deleted == 1
|
||||
assert result.temp_bytes_freed == 500
|
||||
assert old_file.exists()
|
||||
|
||||
|
||||
class TestMultipartCleanup:
|
||||
def test_old_orphaned_multipart_deleted(self, storage_root, gc):
|
||||
bucket = storage_root / "test-bucket"
|
||||
bucket.mkdir()
|
||||
mp_root = storage_root / ".myfsio.sys" / "multipart" / "test-bucket"
|
||||
mp_root.mkdir(parents=True)
|
||||
upload_dir = mp_root / "upload-123"
|
||||
upload_dir.mkdir()
|
||||
manifest = upload_dir / "manifest.json"
|
||||
manifest.write_text(json.dumps({"upload_id": "upload-123", "object_key": "foo.txt"}))
|
||||
part = upload_dir / "part-00001.part"
|
||||
part.write_bytes(b"x" * 2000)
|
||||
_make_old(manifest, hours=200)
|
||||
_make_old(part, hours=200)
|
||||
_make_old(upload_dir, hours=200)
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.multipart_uploads_deleted == 1
|
||||
assert result.multipart_bytes_freed > 0
|
||||
assert not upload_dir.exists()
|
||||
|
||||
def test_recent_multipart_kept(self, storage_root, gc):
|
||||
bucket = storage_root / "test-bucket"
|
||||
bucket.mkdir()
|
||||
mp_root = storage_root / ".myfsio.sys" / "multipart" / "test-bucket"
|
||||
mp_root.mkdir(parents=True)
|
||||
upload_dir = mp_root / "upload-new"
|
||||
upload_dir.mkdir()
|
||||
manifest = upload_dir / "manifest.json"
|
||||
manifest.write_text(json.dumps({"upload_id": "upload-new", "object_key": "bar.txt"}))
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.multipart_uploads_deleted == 0
|
||||
assert upload_dir.exists()
|
||||
|
||||
def test_legacy_multipart_cleaned(self, storage_root, gc):
|
||||
bucket = storage_root / "test-bucket"
|
||||
bucket.mkdir()
|
||||
legacy_mp = bucket / ".multipart" / "upload-old"
|
||||
legacy_mp.mkdir(parents=True)
|
||||
part = legacy_mp / "part-00001.part"
|
||||
part.write_bytes(b"y" * 500)
|
||||
_make_old(part, hours=200)
|
||||
_make_old(legacy_mp, hours=200)
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.multipart_uploads_deleted == 1
|
||||
|
||||
|
||||
class TestLockFileCleanup:
|
||||
def test_stale_lock_files_deleted(self, storage_root, gc):
|
||||
locks_dir = storage_root / ".myfsio.sys" / "buckets" / "test-bucket" / "locks"
|
||||
locks_dir.mkdir(parents=True)
|
||||
lock = locks_dir / "some_key.lock"
|
||||
lock.write_text("")
|
||||
_make_old(lock, hours=2)
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.lock_files_deleted == 1
|
||||
assert not lock.exists()
|
||||
|
||||
def test_recent_lock_kept(self, storage_root, gc):
|
||||
locks_dir = storage_root / ".myfsio.sys" / "buckets" / "test-bucket" / "locks"
|
||||
locks_dir.mkdir(parents=True)
|
||||
lock = locks_dir / "active.lock"
|
||||
lock.write_text("")
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.lock_files_deleted == 0
|
||||
assert lock.exists()
|
||||
|
||||
|
||||
class TestOrphanedMetadataCleanup:
|
||||
def test_legacy_orphaned_metadata_deleted(self, storage_root, gc):
|
||||
bucket = storage_root / "test-bucket"
|
||||
bucket.mkdir()
|
||||
meta_dir = bucket / ".meta"
|
||||
meta_dir.mkdir()
|
||||
orphan = meta_dir / "deleted_file.txt.meta.json"
|
||||
orphan.write_text(json.dumps({"etag": "abc"}))
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.orphaned_metadata_deleted == 1
|
||||
assert not orphan.exists()
|
||||
|
||||
def test_valid_metadata_kept(self, storage_root, gc):
|
||||
bucket = storage_root / "test-bucket"
|
||||
bucket.mkdir()
|
||||
obj = bucket / "exists.txt"
|
||||
obj.write_text("hello")
|
||||
meta_dir = bucket / ".meta"
|
||||
meta_dir.mkdir()
|
||||
meta = meta_dir / "exists.txt.meta.json"
|
||||
meta.write_text(json.dumps({"etag": "abc"}))
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.orphaned_metadata_deleted == 0
|
||||
assert meta.exists()
|
||||
|
||||
def test_index_orphaned_entries_cleaned(self, storage_root, gc):
|
||||
bucket = storage_root / "test-bucket"
|
||||
bucket.mkdir()
|
||||
obj = bucket / "keep.txt"
|
||||
obj.write_text("hello")
|
||||
|
||||
meta_dir = storage_root / ".myfsio.sys" / "buckets" / "test-bucket" / "meta"
|
||||
meta_dir.mkdir(parents=True)
|
||||
index = meta_dir / "_index.json"
|
||||
index.write_text(json.dumps({"keep.txt": {"etag": "a"}, "gone.txt": {"etag": "b"}}))
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.orphaned_metadata_deleted == 1
|
||||
|
||||
updated = json.loads(index.read_text())
|
||||
assert "keep.txt" in updated
|
||||
assert "gone.txt" not in updated
|
||||
|
||||
|
||||
class TestOrphanedVersionsCleanup:
|
||||
def test_orphaned_versions_deleted(self, storage_root, gc):
|
||||
bucket = storage_root / "test-bucket"
|
||||
bucket.mkdir()
|
||||
versions_dir = storage_root / ".myfsio.sys" / "buckets" / "test-bucket" / "versions" / "deleted_obj.txt"
|
||||
versions_dir.mkdir(parents=True)
|
||||
v_bin = versions_dir / "v1.bin"
|
||||
v_json = versions_dir / "v1.json"
|
||||
v_bin.write_bytes(b"old data" * 100)
|
||||
v_json.write_text(json.dumps({"version_id": "v1", "size": 800}))
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.orphaned_versions_deleted == 2
|
||||
assert result.orphaned_version_bytes_freed == 800
|
||||
|
||||
def test_active_versions_kept(self, storage_root, gc):
|
||||
bucket = storage_root / "test-bucket"
|
||||
bucket.mkdir()
|
||||
obj = bucket / "active.txt"
|
||||
obj.write_text("current")
|
||||
versions_dir = storage_root / ".myfsio.sys" / "buckets" / "test-bucket" / "versions" / "active.txt"
|
||||
versions_dir.mkdir(parents=True)
|
||||
v_bin = versions_dir / "v1.bin"
|
||||
v_bin.write_bytes(b"old version")
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.orphaned_versions_deleted == 0
|
||||
assert v_bin.exists()
|
||||
|
||||
|
||||
class TestEmptyDirCleanup:
|
||||
def test_empty_dirs_removed(self, storage_root, gc):
|
||||
empty = storage_root / ".myfsio.sys" / "buckets" / "test-bucket" / "locks" / "sub"
|
||||
empty.mkdir(parents=True)
|
||||
|
||||
result = gc.run_now()
|
||||
assert result.empty_dirs_removed > 0
|
||||
assert not empty.exists()
|
||||
|
||||
|
||||
class TestHistory:
|
||||
def test_history_recorded(self, storage_root, gc):
|
||||
gc.run_now()
|
||||
history = gc.get_history()
|
||||
assert len(history) == 1
|
||||
assert "result" in history[0]
|
||||
assert "timestamp" in history[0]
|
||||
|
||||
def test_multiple_runs(self, storage_root, gc):
|
||||
gc.run_now()
|
||||
gc.run_now()
|
||||
gc.run_now()
|
||||
history = gc.get_history()
|
||||
assert len(history) == 3
|
||||
assert history[0]["timestamp"] >= history[1]["timestamp"]
|
||||
|
||||
|
||||
class TestStatus:
|
||||
def test_get_status(self, storage_root, gc):
|
||||
status = gc.get_status()
|
||||
assert status["interval_hours"] == 1.0
|
||||
assert status["dry_run"] is False
|
||||
assert status["temp_file_max_age_hours"] == 1.0
|
||||
assert status["multipart_max_age_days"] == 1
|
||||
assert status["lock_file_max_age_hours"] == 0.5
|
||||
|
||||
|
||||
class TestGCResult:
|
||||
def test_total_bytes_freed(self):
|
||||
r = GCResult(temp_bytes_freed=100, multipart_bytes_freed=200, orphaned_version_bytes_freed=300)
|
||||
assert r.total_bytes_freed == 600
|
||||
|
||||
def test_has_work(self):
|
||||
assert not GCResult().has_work
|
||||
assert GCResult(temp_files_deleted=1).has_work
|
||||
assert GCResult(lock_files_deleted=1).has_work
|
||||
assert GCResult(empty_dirs_removed=1).has_work
|
||||
|
||||
|
||||
class TestAdminAPI:
|
||||
@pytest.fixture
|
||||
def gc_app(self, tmp_path):
|
||||
from app import create_api_app
|
||||
storage_root = tmp_path / "data"
|
||||
iam_config = tmp_path / "iam.json"
|
||||
bucket_policies = tmp_path / "bucket_policies.json"
|
||||
iam_payload = {
|
||||
"users": [
|
||||
{
|
||||
"access_key": "admin",
|
||||
"secret_key": "adminsecret",
|
||||
"display_name": "Admin",
|
||||
"policies": [{"bucket": "*", "actions": ["list", "read", "write", "delete", "policy", "iam:*"]}],
|
||||
}
|
||||
]
|
||||
}
|
||||
iam_config.write_text(json.dumps(iam_payload))
|
||||
flask_app = create_api_app({
|
||||
"TESTING": True,
|
||||
"SECRET_KEY": "testing",
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"GC_ENABLED": True,
|
||||
"GC_INTERVAL_HOURS": 1.0,
|
||||
})
|
||||
yield flask_app
|
||||
gc = flask_app.extensions.get("gc")
|
||||
if gc:
|
||||
gc.stop()
|
||||
|
||||
def test_gc_status(self, gc_app):
|
||||
client = gc_app.test_client()
|
||||
resp = client.get("/admin/gc/status", headers={"X-Access-Key": "admin", "X-Secret-Key": "adminsecret"})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["enabled"] is True
|
||||
|
||||
def test_gc_run(self, gc_app):
|
||||
client = gc_app.test_client()
|
||||
resp = client.post(
|
||||
"/admin/gc/run",
|
||||
headers={"X-Access-Key": "admin", "X-Secret-Key": "adminsecret"},
|
||||
content_type="application/json",
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["status"] == "started"
|
||||
|
||||
def test_gc_dry_run(self, gc_app):
|
||||
client = gc_app.test_client()
|
||||
resp = client.post(
|
||||
"/admin/gc/run",
|
||||
headers={"X-Access-Key": "admin", "X-Secret-Key": "adminsecret"},
|
||||
data=json.dumps({"dry_run": True}),
|
||||
content_type="application/json",
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["status"] == "started"
|
||||
|
||||
def test_gc_history(self, gc_app):
|
||||
import time
|
||||
client = gc_app.test_client()
|
||||
client.post("/admin/gc/run", headers={"X-Access-Key": "admin", "X-Secret-Key": "adminsecret"})
|
||||
for _ in range(50):
|
||||
time.sleep(0.1)
|
||||
status = client.get("/admin/gc/status", headers={"X-Access-Key": "admin", "X-Secret-Key": "adminsecret"}).get_json()
|
||||
if not status.get("scanning"):
|
||||
break
|
||||
resp = client.get("/admin/gc/history", headers={"X-Access-Key": "admin", "X-Secret-Key": "adminsecret"})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert len(data["executions"]) >= 1
|
||||
|
||||
def test_gc_requires_admin(self, gc_app):
|
||||
iam = gc_app.extensions["iam"]
|
||||
user = iam.create_user(display_name="Regular")
|
||||
client = gc_app.test_client()
|
||||
resp = client.get(
|
||||
"/admin/gc/status",
|
||||
headers={"X-Access-Key": user["access_key"], "X-Secret-Key": user["secret_key"]},
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
648
tests/test_integrity.py
Normal file
648
tests/test_integrity.py
Normal file
@@ -0,0 +1,648 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.integrity import IntegrityChecker, IntegrityCursorStore, IntegrityResult
|
||||
|
||||
|
||||
def _wait_scan_done(client, headers, timeout=10):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
resp = client.get("/admin/integrity/status", headers=headers)
|
||||
data = resp.get_json()
|
||||
if not data.get("scanning"):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError("scan did not complete")
|
||||
|
||||
|
||||
def _md5(data: bytes) -> str:
|
||||
return hashlib.md5(data).hexdigest()
|
||||
|
||||
|
||||
def _setup_bucket(storage_root: Path, bucket_name: str, objects: dict[str, bytes]) -> None:
|
||||
bucket_path = storage_root / bucket_name
|
||||
bucket_path.mkdir(parents=True, exist_ok=True)
|
||||
meta_root = storage_root / ".myfsio.sys" / "buckets" / bucket_name / "meta"
|
||||
meta_root.mkdir(parents=True, exist_ok=True)
|
||||
bucket_json = storage_root / ".myfsio.sys" / "buckets" / bucket_name / ".bucket.json"
|
||||
bucket_json.write_text(json.dumps({"created": "2025-01-01"}))
|
||||
|
||||
for key, data in objects.items():
|
||||
obj_path = bucket_path / key
|
||||
obj_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
obj_path.write_bytes(data)
|
||||
|
||||
etag = _md5(data)
|
||||
stat = obj_path.stat()
|
||||
meta = {
|
||||
"__etag__": etag,
|
||||
"__size__": str(stat.st_size),
|
||||
"__last_modified__": str(stat.st_mtime),
|
||||
}
|
||||
|
||||
key_path = Path(key)
|
||||
parent = key_path.parent
|
||||
key_name = key_path.name
|
||||
if parent == Path("."):
|
||||
index_path = meta_root / "_index.json"
|
||||
else:
|
||||
index_path = meta_root / parent / "_index.json"
|
||||
index_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
index_data = {}
|
||||
if index_path.exists():
|
||||
index_data = json.loads(index_path.read_text())
|
||||
index_data[key_name] = {"metadata": meta}
|
||||
index_path.write_text(json.dumps(index_data))
|
||||
|
||||
|
||||
def _issues_of_type(result, issue_type):
|
||||
return [i for i in result.issues if i.issue_type == issue_type]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage_root(tmp_path):
|
||||
root = tmp_path / "data"
|
||||
root.mkdir()
|
||||
(root / ".myfsio.sys" / "config").mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def checker(storage_root):
|
||||
return IntegrityChecker(
|
||||
storage_root=storage_root,
|
||||
interval_hours=24.0,
|
||||
batch_size=1000,
|
||||
auto_heal=False,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
|
||||
class TestCorruptedObjects:
|
||||
def test_detect_corrupted(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello world"})
|
||||
(storage_root / "mybucket" / "file.txt").write_bytes(b"corrupted data")
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.corrupted_objects == 1
|
||||
issues = _issues_of_type(result, "corrupted_object")
|
||||
assert len(issues) == 1
|
||||
assert issues[0].bucket == "mybucket"
|
||||
assert issues[0].key == "file.txt"
|
||||
assert not issues[0].healed
|
||||
|
||||
def test_heal_corrupted(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello world"})
|
||||
(storage_root / "mybucket" / "file.txt").write_bytes(b"corrupted data")
|
||||
|
||||
result = checker.run_now(auto_heal=True)
|
||||
assert result.corrupted_objects == 1
|
||||
assert result.issues_healed == 1
|
||||
issues = _issues_of_type(result, "corrupted_object")
|
||||
assert issues[0].healed
|
||||
|
||||
result2 = checker.run_now()
|
||||
assert result2.corrupted_objects == 0
|
||||
|
||||
def test_valid_objects_pass(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello world"})
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.corrupted_objects == 0
|
||||
assert result.objects_scanned >= 1
|
||||
|
||||
def test_corrupted_nested_key(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"sub/dir/file.txt": b"nested content"})
|
||||
(storage_root / "mybucket" / "sub" / "dir" / "file.txt").write_bytes(b"bad")
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.corrupted_objects == 1
|
||||
issues = _issues_of_type(result, "corrupted_object")
|
||||
assert issues[0].key == "sub/dir/file.txt"
|
||||
|
||||
|
||||
class TestOrphanedObjects:
|
||||
def test_detect_orphaned(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {})
|
||||
(storage_root / "mybucket" / "orphan.txt").write_bytes(b"orphan data")
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.orphaned_objects == 1
|
||||
issues = _issues_of_type(result, "orphaned_object")
|
||||
assert len(issues) == 1
|
||||
|
||||
def test_heal_orphaned(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {})
|
||||
(storage_root / "mybucket" / "orphan.txt").write_bytes(b"orphan data")
|
||||
|
||||
result = checker.run_now(auto_heal=True)
|
||||
assert result.orphaned_objects == 1
|
||||
assert result.issues_healed == 1
|
||||
issues = _issues_of_type(result, "orphaned_object")
|
||||
assert issues[0].healed
|
||||
|
||||
result2 = checker.run_now()
|
||||
assert result2.orphaned_objects == 0
|
||||
assert result2.objects_scanned >= 1
|
||||
|
||||
|
||||
class TestPhantomMetadata:
|
||||
def test_detect_phantom(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
(storage_root / "mybucket" / "file.txt").unlink()
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.phantom_metadata == 1
|
||||
issues = _issues_of_type(result, "phantom_metadata")
|
||||
assert len(issues) == 1
|
||||
|
||||
def test_heal_phantom(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
(storage_root / "mybucket" / "file.txt").unlink()
|
||||
|
||||
result = checker.run_now(auto_heal=True)
|
||||
assert result.phantom_metadata == 1
|
||||
assert result.issues_healed == 1
|
||||
|
||||
result2 = checker.run_now()
|
||||
assert result2.phantom_metadata == 0
|
||||
|
||||
|
||||
class TestStaleVersions:
|
||||
def test_manifest_without_data(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
versions_root = storage_root / ".myfsio.sys" / "buckets" / "mybucket" / "versions" / "file.txt"
|
||||
versions_root.mkdir(parents=True)
|
||||
(versions_root / "v1.json").write_text(json.dumps({"etag": "abc"}))
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.stale_versions == 1
|
||||
issues = _issues_of_type(result, "stale_version")
|
||||
assert "manifest without data" in issues[0].detail
|
||||
|
||||
def test_data_without_manifest(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
versions_root = storage_root / ".myfsio.sys" / "buckets" / "mybucket" / "versions" / "file.txt"
|
||||
versions_root.mkdir(parents=True)
|
||||
(versions_root / "v1.bin").write_bytes(b"old data")
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.stale_versions == 1
|
||||
issues = _issues_of_type(result, "stale_version")
|
||||
assert "data without manifest" in issues[0].detail
|
||||
|
||||
def test_heal_stale_versions(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
versions_root = storage_root / ".myfsio.sys" / "buckets" / "mybucket" / "versions" / "file.txt"
|
||||
versions_root.mkdir(parents=True)
|
||||
(versions_root / "v1.json").write_text(json.dumps({"etag": "abc"}))
|
||||
(versions_root / "v2.bin").write_bytes(b"old data")
|
||||
|
||||
result = checker.run_now(auto_heal=True)
|
||||
assert result.stale_versions == 2
|
||||
assert result.issues_healed == 2
|
||||
assert not (versions_root / "v1.json").exists()
|
||||
assert not (versions_root / "v2.bin").exists()
|
||||
|
||||
def test_valid_versions_pass(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
versions_root = storage_root / ".myfsio.sys" / "buckets" / "mybucket" / "versions" / "file.txt"
|
||||
versions_root.mkdir(parents=True)
|
||||
(versions_root / "v1.json").write_text(json.dumps({"etag": "abc"}))
|
||||
(versions_root / "v1.bin").write_bytes(b"old data")
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.stale_versions == 0
|
||||
|
||||
|
||||
class TestEtagCache:
|
||||
def test_detect_mismatch(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
etag_path = storage_root / ".myfsio.sys" / "buckets" / "mybucket" / "etag_index.json"
|
||||
etag_path.write_text(json.dumps({"file.txt": "wrong_etag"}))
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.etag_cache_inconsistencies == 1
|
||||
issues = _issues_of_type(result, "etag_cache_inconsistency")
|
||||
assert len(issues) == 1
|
||||
|
||||
def test_heal_mismatch(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
etag_path = storage_root / ".myfsio.sys" / "buckets" / "mybucket" / "etag_index.json"
|
||||
etag_path.write_text(json.dumps({"file.txt": "wrong_etag"}))
|
||||
|
||||
result = checker.run_now(auto_heal=True)
|
||||
assert result.etag_cache_inconsistencies == 1
|
||||
assert result.issues_healed == 1
|
||||
assert not etag_path.exists()
|
||||
|
||||
|
||||
class TestLegacyMetadata:
|
||||
def test_detect_unmigrated(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
|
||||
legacy_meta = storage_root / "mybucket" / ".meta" / "file.txt.meta.json"
|
||||
legacy_meta.parent.mkdir(parents=True)
|
||||
legacy_meta.write_text(json.dumps({"__etag__": "different_value"}))
|
||||
|
||||
meta_root = storage_root / ".myfsio.sys" / "buckets" / "mybucket" / "meta"
|
||||
index_path = meta_root / "_index.json"
|
||||
index_path.unlink()
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.legacy_metadata_drifts == 1
|
||||
issues = _issues_of_type(result, "legacy_metadata_drift")
|
||||
assert len(issues) == 1
|
||||
assert issues[0].detail == "unmigrated legacy .meta.json"
|
||||
|
||||
def test_detect_drift(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
|
||||
legacy_meta = storage_root / "mybucket" / ".meta" / "file.txt.meta.json"
|
||||
legacy_meta.parent.mkdir(parents=True)
|
||||
legacy_meta.write_text(json.dumps({"__etag__": "different_value"}))
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.legacy_metadata_drifts == 1
|
||||
issues = _issues_of_type(result, "legacy_metadata_drift")
|
||||
assert "differs from index" in issues[0].detail
|
||||
|
||||
def test_heal_unmigrated(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
|
||||
legacy_meta = storage_root / "mybucket" / ".meta" / "file.txt.meta.json"
|
||||
legacy_meta.parent.mkdir(parents=True)
|
||||
legacy_data = {"__etag__": _md5(b"hello"), "__size__": "5"}
|
||||
legacy_meta.write_text(json.dumps(legacy_data))
|
||||
|
||||
meta_root = storage_root / ".myfsio.sys" / "buckets" / "mybucket" / "meta"
|
||||
index_path = meta_root / "_index.json"
|
||||
index_path.unlink()
|
||||
|
||||
result = checker.run_now(auto_heal=True)
|
||||
assert result.legacy_metadata_drifts == 1
|
||||
legacy_issues = _issues_of_type(result, "legacy_metadata_drift")
|
||||
assert len(legacy_issues) == 1
|
||||
assert legacy_issues[0].healed
|
||||
assert not legacy_meta.exists()
|
||||
|
||||
index_data = json.loads(index_path.read_text())
|
||||
assert "file.txt" in index_data
|
||||
assert index_data["file.txt"]["metadata"]["__etag__"] == _md5(b"hello")
|
||||
|
||||
def test_heal_drift(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
|
||||
legacy_meta = storage_root / "mybucket" / ".meta" / "file.txt.meta.json"
|
||||
legacy_meta.parent.mkdir(parents=True)
|
||||
legacy_meta.write_text(json.dumps({"__etag__": "different_value"}))
|
||||
|
||||
result = checker.run_now(auto_heal=True)
|
||||
assert result.legacy_metadata_drifts == 1
|
||||
legacy_issues = _issues_of_type(result, "legacy_metadata_drift")
|
||||
assert legacy_issues[0].healed
|
||||
assert not legacy_meta.exists()
|
||||
|
||||
|
||||
class TestDryRun:
|
||||
def test_dry_run_no_changes(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
(storage_root / "mybucket" / "file.txt").write_bytes(b"corrupted")
|
||||
(storage_root / "mybucket" / "orphan.txt").write_bytes(b"orphan")
|
||||
|
||||
result = checker.run_now(auto_heal=True, dry_run=True)
|
||||
assert result.corrupted_objects == 1
|
||||
assert result.orphaned_objects == 1
|
||||
assert result.issues_healed == 0
|
||||
|
||||
meta_root = storage_root / ".myfsio.sys" / "buckets" / "mybucket" / "meta"
|
||||
index_data = json.loads((meta_root / "_index.json").read_text())
|
||||
assert "orphan.txt" not in index_data
|
||||
|
||||
|
||||
class TestBatchSize:
|
||||
def test_batch_limits_scan(self, storage_root):
|
||||
objects = {f"file{i}.txt": f"data{i}".encode() for i in range(10)}
|
||||
_setup_bucket(storage_root, "mybucket", objects)
|
||||
|
||||
checker = IntegrityChecker(
|
||||
storage_root=storage_root,
|
||||
batch_size=3,
|
||||
)
|
||||
result = checker.run_now()
|
||||
assert result.objects_scanned <= 3
|
||||
|
||||
|
||||
class TestHistory:
|
||||
def test_history_recorded(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
|
||||
checker.run_now()
|
||||
history = checker.get_history()
|
||||
assert len(history) == 1
|
||||
assert "corrupted_objects" in history[0]["result"]
|
||||
|
||||
def test_history_multiple(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
|
||||
checker.run_now()
|
||||
checker.run_now()
|
||||
checker.run_now()
|
||||
history = checker.get_history()
|
||||
assert len(history) == 3
|
||||
|
||||
def test_history_pagination(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
|
||||
for _ in range(5):
|
||||
checker.run_now()
|
||||
|
||||
history = checker.get_history(limit=2, offset=1)
|
||||
assert len(history) == 2
|
||||
|
||||
|
||||
AUTH_HEADERS = {"X-Access-Key": "admin", "X-Secret-Key": "adminsecret"}
|
||||
|
||||
|
||||
class TestAdminAPI:
|
||||
@pytest.fixture
|
||||
def integrity_app(self, tmp_path):
|
||||
from app import create_api_app
|
||||
storage_root = tmp_path / "data"
|
||||
iam_config = tmp_path / "iam.json"
|
||||
bucket_policies = tmp_path / "bucket_policies.json"
|
||||
iam_payload = {
|
||||
"users": [
|
||||
{
|
||||
"access_key": "admin",
|
||||
"secret_key": "adminsecret",
|
||||
"display_name": "Admin",
|
||||
"policies": [{"bucket": "*", "actions": ["list", "read", "write", "delete", "policy", "iam:*"]}],
|
||||
}
|
||||
]
|
||||
}
|
||||
iam_config.write_text(json.dumps(iam_payload))
|
||||
flask_app = create_api_app({
|
||||
"TESTING": True,
|
||||
"SECRET_KEY": "testing",
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"INTEGRITY_ENABLED": True,
|
||||
"INTEGRITY_AUTO_HEAL": False,
|
||||
"INTEGRITY_DRY_RUN": False,
|
||||
})
|
||||
yield flask_app
|
||||
storage = flask_app.extensions.get("object_storage")
|
||||
if storage:
|
||||
base = getattr(storage, "storage", storage)
|
||||
if hasattr(base, "shutdown_stats"):
|
||||
base.shutdown_stats()
|
||||
ic = flask_app.extensions.get("integrity")
|
||||
if ic:
|
||||
ic.stop()
|
||||
|
||||
def test_status_endpoint(self, integrity_app):
|
||||
client = integrity_app.test_client()
|
||||
resp = client.get("/admin/integrity/status", headers=AUTH_HEADERS)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["enabled"] is True
|
||||
assert "interval_hours" in data
|
||||
|
||||
def test_run_endpoint(self, integrity_app):
|
||||
client = integrity_app.test_client()
|
||||
resp = client.post("/admin/integrity/run", headers=AUTH_HEADERS, json={})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["status"] == "started"
|
||||
_wait_scan_done(client, AUTH_HEADERS)
|
||||
resp = client.get("/admin/integrity/history?limit=1", headers=AUTH_HEADERS)
|
||||
hist = resp.get_json()
|
||||
assert len(hist["executions"]) >= 1
|
||||
assert "corrupted_objects" in hist["executions"][0]["result"]
|
||||
assert "objects_scanned" in hist["executions"][0]["result"]
|
||||
|
||||
def test_run_with_overrides(self, integrity_app):
|
||||
client = integrity_app.test_client()
|
||||
resp = client.post(
|
||||
"/admin/integrity/run",
|
||||
headers=AUTH_HEADERS,
|
||||
json={"dry_run": True, "auto_heal": True},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
_wait_scan_done(client, AUTH_HEADERS)
|
||||
|
||||
def test_history_endpoint(self, integrity_app):
|
||||
client = integrity_app.test_client()
|
||||
client.post("/admin/integrity/run", headers=AUTH_HEADERS, json={})
|
||||
_wait_scan_done(client, AUTH_HEADERS)
|
||||
resp = client.get("/admin/integrity/history", headers=AUTH_HEADERS)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert "executions" in data
|
||||
assert len(data["executions"]) >= 1
|
||||
|
||||
def test_auth_required(self, integrity_app):
|
||||
client = integrity_app.test_client()
|
||||
resp = client.get("/admin/integrity/status")
|
||||
assert resp.status_code in (401, 403)
|
||||
|
||||
def test_disabled_status(self, tmp_path):
|
||||
from app import create_api_app
|
||||
storage_root = tmp_path / "data2"
|
||||
iam_config = tmp_path / "iam2.json"
|
||||
bucket_policies = tmp_path / "bp2.json"
|
||||
iam_payload = {
|
||||
"users": [
|
||||
{
|
||||
"access_key": "admin",
|
||||
"secret_key": "adminsecret",
|
||||
"display_name": "Admin",
|
||||
"policies": [{"bucket": "*", "actions": ["list", "read", "write", "delete", "policy", "iam:*"]}],
|
||||
}
|
||||
]
|
||||
}
|
||||
iam_config.write_text(json.dumps(iam_payload))
|
||||
flask_app = create_api_app({
|
||||
"TESTING": True,
|
||||
"SECRET_KEY": "testing",
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"INTEGRITY_ENABLED": False,
|
||||
})
|
||||
c = flask_app.test_client()
|
||||
resp = c.get("/admin/integrity/status", headers=AUTH_HEADERS)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["enabled"] is False
|
||||
|
||||
storage = flask_app.extensions.get("object_storage")
|
||||
if storage:
|
||||
base = getattr(storage, "storage", storage)
|
||||
if hasattr(base, "shutdown_stats"):
|
||||
base.shutdown_stats()
|
||||
|
||||
|
||||
class TestMultipleBuckets:
|
||||
def test_scans_multiple_buckets(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "bucket1", {"a.txt": b"aaa"})
|
||||
_setup_bucket(storage_root, "bucket2", {"b.txt": b"bbb"})
|
||||
|
||||
result = checker.run_now()
|
||||
assert result.buckets_scanned == 2
|
||||
assert result.objects_scanned >= 2
|
||||
assert result.corrupted_objects == 0
|
||||
|
||||
|
||||
class TestGetStatus:
|
||||
def test_status_fields(self, checker):
|
||||
status = checker.get_status()
|
||||
assert "enabled" in status
|
||||
assert "running" in status
|
||||
assert "interval_hours" in status
|
||||
assert "batch_size" in status
|
||||
assert "auto_heal" in status
|
||||
assert "dry_run" in status
|
||||
|
||||
def test_status_includes_cursor(self, storage_root, checker):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
checker.run_now()
|
||||
status = checker.get_status()
|
||||
assert "cursor" in status
|
||||
assert status["cursor"]["tracked_buckets"] == 1
|
||||
assert "mybucket" in status["cursor"]["buckets"]
|
||||
|
||||
|
||||
class TestUnifiedBatchCounter:
|
||||
def test_orphaned_objects_count_toward_batch(self, storage_root):
|
||||
_setup_bucket(storage_root, "mybucket", {})
|
||||
for i in range(10):
|
||||
(storage_root / "mybucket" / f"orphan{i}.txt").write_bytes(f"data{i}".encode())
|
||||
|
||||
checker = IntegrityChecker(storage_root=storage_root, batch_size=3)
|
||||
result = checker.run_now()
|
||||
assert result.objects_scanned <= 3
|
||||
|
||||
def test_phantom_metadata_counts_toward_batch(self, storage_root):
|
||||
objects = {f"file{i}.txt": f"data{i}".encode() for i in range(10)}
|
||||
_setup_bucket(storage_root, "mybucket", objects)
|
||||
for i in range(10):
|
||||
(storage_root / "mybucket" / f"file{i}.txt").unlink()
|
||||
|
||||
checker = IntegrityChecker(storage_root=storage_root, batch_size=5)
|
||||
result = checker.run_now()
|
||||
assert result.objects_scanned <= 5
|
||||
|
||||
def test_all_check_types_contribute(self, storage_root):
|
||||
_setup_bucket(storage_root, "mybucket", {"valid.txt": b"hello"})
|
||||
(storage_root / "mybucket" / "orphan.txt").write_bytes(b"orphan")
|
||||
|
||||
checker = IntegrityChecker(storage_root=storage_root, batch_size=1000)
|
||||
result = checker.run_now()
|
||||
assert result.objects_scanned > 2
|
||||
|
||||
|
||||
class TestCursorRotation:
|
||||
def test_oldest_bucket_scanned_first(self, storage_root):
|
||||
_setup_bucket(storage_root, "bucket-a", {"a.txt": b"aaa"})
|
||||
_setup_bucket(storage_root, "bucket-b", {"b.txt": b"bbb"})
|
||||
_setup_bucket(storage_root, "bucket-c", {"c.txt": b"ccc"})
|
||||
|
||||
checker = IntegrityChecker(storage_root=storage_root, batch_size=5)
|
||||
|
||||
checker.cursor_store.update_bucket("bucket-a", 1000.0)
|
||||
checker.cursor_store.update_bucket("bucket-b", 3000.0)
|
||||
checker.cursor_store.update_bucket("bucket-c", 2000.0)
|
||||
|
||||
ordered = checker.cursor_store.get_bucket_order(["bucket-a", "bucket-b", "bucket-c"])
|
||||
assert ordered[0] == "bucket-a"
|
||||
assert ordered[1] == "bucket-c"
|
||||
assert ordered[2] == "bucket-b"
|
||||
|
||||
def test_never_scanned_buckets_first(self, storage_root):
|
||||
_setup_bucket(storage_root, "bucket-old", {"a.txt": b"aaa"})
|
||||
_setup_bucket(storage_root, "bucket-new", {"b.txt": b"bbb"})
|
||||
|
||||
checker = IntegrityChecker(storage_root=storage_root, batch_size=1000)
|
||||
|
||||
checker.cursor_store.update_bucket("bucket-old", time.time())
|
||||
|
||||
ordered = checker.cursor_store.get_bucket_order(["bucket-old", "bucket-new"])
|
||||
assert ordered[0] == "bucket-new"
|
||||
|
||||
def test_rotation_covers_all_buckets(self, storage_root):
|
||||
for name in ["bucket-a", "bucket-b", "bucket-c"]:
|
||||
_setup_bucket(storage_root, name, {f"{name}.txt": name.encode()})
|
||||
|
||||
checker = IntegrityChecker(storage_root=storage_root, batch_size=4)
|
||||
|
||||
result1 = checker.run_now()
|
||||
scanned_buckets_1 = set()
|
||||
for issue_bucket in [storage_root]:
|
||||
pass
|
||||
assert result1.buckets_scanned >= 1
|
||||
|
||||
result2 = checker.run_now()
|
||||
result3 = checker.run_now()
|
||||
|
||||
cursor_info = checker.cursor_store.get_info()
|
||||
assert cursor_info["tracked_buckets"] == 3
|
||||
|
||||
def test_cursor_persistence(self, storage_root):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
|
||||
checker1 = IntegrityChecker(storage_root=storage_root, batch_size=1000)
|
||||
checker1.run_now()
|
||||
|
||||
cursor1 = checker1.cursor_store.get_info()
|
||||
assert cursor1["tracked_buckets"] == 1
|
||||
assert "mybucket" in cursor1["buckets"]
|
||||
|
||||
checker2 = IntegrityChecker(storage_root=storage_root, batch_size=1000)
|
||||
cursor2 = checker2.cursor_store.get_info()
|
||||
assert cursor2["tracked_buckets"] == 1
|
||||
assert "mybucket" in cursor2["buckets"]
|
||||
|
||||
def test_stale_cursor_cleanup(self, storage_root):
|
||||
_setup_bucket(storage_root, "bucket-a", {"a.txt": b"aaa"})
|
||||
_setup_bucket(storage_root, "bucket-b", {"b.txt": b"bbb"})
|
||||
|
||||
checker = IntegrityChecker(storage_root=storage_root, batch_size=1000)
|
||||
checker.run_now()
|
||||
|
||||
import shutil
|
||||
shutil.rmtree(storage_root / "bucket-b")
|
||||
meta_b = storage_root / ".myfsio.sys" / "buckets" / "bucket-b"
|
||||
if meta_b.exists():
|
||||
shutil.rmtree(meta_b)
|
||||
|
||||
checker.run_now()
|
||||
|
||||
cursor_info = checker.cursor_store.get_info()
|
||||
assert "bucket-b" not in cursor_info["buckets"]
|
||||
assert "bucket-a" in cursor_info["buckets"]
|
||||
|
||||
def test_cursor_updates_after_scan(self, storage_root):
|
||||
_setup_bucket(storage_root, "mybucket", {"file.txt": b"hello"})
|
||||
|
||||
checker = IntegrityChecker(storage_root=storage_root, batch_size=1000)
|
||||
before = time.time()
|
||||
checker.run_now()
|
||||
after = time.time()
|
||||
|
||||
cursor_info = checker.cursor_store.get_info()
|
||||
ts = cursor_info["buckets"]["mybucket"]
|
||||
assert before <= ts <= after
|
||||
@@ -321,8 +321,9 @@ class TestNotificationService:
|
||||
assert "events_sent" in stats
|
||||
assert "events_failed" in stats
|
||||
|
||||
@patch("app.notifications.requests.post")
|
||||
def test_send_notification_success(self, mock_post, notification_service):
|
||||
@patch("app.notifications._pinned_post")
|
||||
@patch("app.notifications._resolve_and_check_url", return_value="93.184.216.34")
|
||||
def test_send_notification_success(self, mock_resolve, mock_post, notification_service):
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_post.return_value = mock_response
|
||||
@@ -337,8 +338,9 @@ class TestNotificationService:
|
||||
notification_service._send_notification(event, destination)
|
||||
mock_post.assert_called_once()
|
||||
|
||||
@patch("app.notifications.requests.post")
|
||||
def test_send_notification_retry_on_failure(self, mock_post, notification_service):
|
||||
@patch("app.notifications._pinned_post")
|
||||
@patch("app.notifications._resolve_and_check_url", return_value="93.184.216.34")
|
||||
def test_send_notification_retry_on_failure(self, mock_resolve, mock_post, notification_service):
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 500
|
||||
mock_response.text = "Internal Server Error"
|
||||
|
||||
350
tests/test_rust_extensions.py
Normal file
350
tests/test_rust_extensions.py
Normal file
@@ -0,0 +1,350 @@
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
try:
|
||||
import myfsio_core as _rc
|
||||
HAS_RUST = True
|
||||
except ImportError:
|
||||
_rc = None
|
||||
HAS_RUST = False
|
||||
|
||||
pytestmark = pytest.mark.skipif(not HAS_RUST, reason="myfsio_core not available")
|
||||
|
||||
|
||||
class TestStreamToFileWithMd5:
|
||||
def test_basic_write(self, tmp_path):
|
||||
data = b"hello world" * 1000
|
||||
stream = io.BytesIO(data)
|
||||
tmp_dir = str(tmp_path / "tmp")
|
||||
|
||||
tmp_path_str, md5_hex, size = _rc.stream_to_file_with_md5(stream, tmp_dir)
|
||||
|
||||
assert size == len(data)
|
||||
assert md5_hex == hashlib.md5(data).hexdigest()
|
||||
assert Path(tmp_path_str).exists()
|
||||
assert Path(tmp_path_str).read_bytes() == data
|
||||
|
||||
def test_empty_stream(self, tmp_path):
|
||||
stream = io.BytesIO(b"")
|
||||
tmp_dir = str(tmp_path / "tmp")
|
||||
|
||||
tmp_path_str, md5_hex, size = _rc.stream_to_file_with_md5(stream, tmp_dir)
|
||||
|
||||
assert size == 0
|
||||
assert md5_hex == hashlib.md5(b"").hexdigest()
|
||||
assert Path(tmp_path_str).read_bytes() == b""
|
||||
|
||||
def test_large_data(self, tmp_path):
|
||||
data = os.urandom(1024 * 1024 * 2)
|
||||
stream = io.BytesIO(data)
|
||||
tmp_dir = str(tmp_path / "tmp")
|
||||
|
||||
tmp_path_str, md5_hex, size = _rc.stream_to_file_with_md5(stream, tmp_dir)
|
||||
|
||||
assert size == len(data)
|
||||
assert md5_hex == hashlib.md5(data).hexdigest()
|
||||
|
||||
def test_custom_chunk_size(self, tmp_path):
|
||||
data = b"x" * 10000
|
||||
stream = io.BytesIO(data)
|
||||
tmp_dir = str(tmp_path / "tmp")
|
||||
|
||||
tmp_path_str, md5_hex, size = _rc.stream_to_file_with_md5(
|
||||
stream, tmp_dir, chunk_size=128
|
||||
)
|
||||
|
||||
assert size == len(data)
|
||||
assert md5_hex == hashlib.md5(data).hexdigest()
|
||||
|
||||
|
||||
class TestAssemblePartsWithMd5:
|
||||
def test_basic_assembly(self, tmp_path):
|
||||
parts = []
|
||||
combined = b""
|
||||
for i in range(3):
|
||||
data = f"part{i}data".encode() * 100
|
||||
combined += data
|
||||
p = tmp_path / f"part{i}"
|
||||
p.write_bytes(data)
|
||||
parts.append(str(p))
|
||||
|
||||
dest = str(tmp_path / "output")
|
||||
md5_hex = _rc.assemble_parts_with_md5(parts, dest)
|
||||
|
||||
assert md5_hex == hashlib.md5(combined).hexdigest()
|
||||
assert Path(dest).read_bytes() == combined
|
||||
|
||||
def test_single_part(self, tmp_path):
|
||||
data = b"single part data"
|
||||
p = tmp_path / "part0"
|
||||
p.write_bytes(data)
|
||||
|
||||
dest = str(tmp_path / "output")
|
||||
md5_hex = _rc.assemble_parts_with_md5([str(p)], dest)
|
||||
|
||||
assert md5_hex == hashlib.md5(data).hexdigest()
|
||||
assert Path(dest).read_bytes() == data
|
||||
|
||||
def test_empty_parts_list(self):
|
||||
with pytest.raises(ValueError, match="No parts"):
|
||||
_rc.assemble_parts_with_md5([], "dummy")
|
||||
|
||||
def test_missing_part_file(self, tmp_path):
|
||||
with pytest.raises(OSError):
|
||||
_rc.assemble_parts_with_md5(
|
||||
[str(tmp_path / "nonexistent")], str(tmp_path / "out")
|
||||
)
|
||||
|
||||
def test_large_parts(self, tmp_path):
|
||||
parts = []
|
||||
combined = b""
|
||||
for i in range(5):
|
||||
data = os.urandom(512 * 1024)
|
||||
combined += data
|
||||
p = tmp_path / f"part{i}"
|
||||
p.write_bytes(data)
|
||||
parts.append(str(p))
|
||||
|
||||
dest = str(tmp_path / "output")
|
||||
md5_hex = _rc.assemble_parts_with_md5(parts, dest)
|
||||
|
||||
assert md5_hex == hashlib.md5(combined).hexdigest()
|
||||
assert Path(dest).read_bytes() == combined
|
||||
|
||||
|
||||
class TestEncryptDecryptStreamChunked:
|
||||
def _python_derive_chunk_nonce(self, base_nonce, chunk_index):
|
||||
from cryptography.hazmat.primitives.kdf.hkdf import HKDF
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
hkdf = HKDF(
|
||||
algorithm=hashes.SHA256(),
|
||||
length=12,
|
||||
salt=base_nonce,
|
||||
info=chunk_index.to_bytes(4, "big"),
|
||||
)
|
||||
return hkdf.derive(b"chunk_nonce")
|
||||
|
||||
def test_encrypt_decrypt_roundtrip(self, tmp_path):
|
||||
data = b"Hello, encryption!" * 500
|
||||
key = secrets.token_bytes(32)
|
||||
base_nonce = secrets.token_bytes(12)
|
||||
|
||||
input_path = str(tmp_path / "plaintext")
|
||||
encrypted_path = str(tmp_path / "encrypted")
|
||||
decrypted_path = str(tmp_path / "decrypted")
|
||||
|
||||
Path(input_path).write_bytes(data)
|
||||
|
||||
chunk_count = _rc.encrypt_stream_chunked(
|
||||
input_path, encrypted_path, key, base_nonce
|
||||
)
|
||||
assert chunk_count > 0
|
||||
|
||||
chunk_count_dec = _rc.decrypt_stream_chunked(
|
||||
encrypted_path, decrypted_path, key, base_nonce
|
||||
)
|
||||
assert chunk_count_dec == chunk_count
|
||||
assert Path(decrypted_path).read_bytes() == data
|
||||
|
||||
def test_empty_file(self, tmp_path):
|
||||
key = secrets.token_bytes(32)
|
||||
base_nonce = secrets.token_bytes(12)
|
||||
|
||||
input_path = str(tmp_path / "empty")
|
||||
encrypted_path = str(tmp_path / "encrypted")
|
||||
decrypted_path = str(tmp_path / "decrypted")
|
||||
|
||||
Path(input_path).write_bytes(b"")
|
||||
|
||||
chunk_count = _rc.encrypt_stream_chunked(
|
||||
input_path, encrypted_path, key, base_nonce
|
||||
)
|
||||
assert chunk_count == 0
|
||||
|
||||
chunk_count_dec = _rc.decrypt_stream_chunked(
|
||||
encrypted_path, decrypted_path, key, base_nonce
|
||||
)
|
||||
assert chunk_count_dec == 0
|
||||
assert Path(decrypted_path).read_bytes() == b""
|
||||
|
||||
def test_custom_chunk_size(self, tmp_path):
|
||||
data = os.urandom(10000)
|
||||
key = secrets.token_bytes(32)
|
||||
base_nonce = secrets.token_bytes(12)
|
||||
|
||||
input_path = str(tmp_path / "plaintext")
|
||||
encrypted_path = str(tmp_path / "encrypted")
|
||||
decrypted_path = str(tmp_path / "decrypted")
|
||||
|
||||
Path(input_path).write_bytes(data)
|
||||
|
||||
chunk_count = _rc.encrypt_stream_chunked(
|
||||
input_path, encrypted_path, key, base_nonce, chunk_size=1024
|
||||
)
|
||||
assert chunk_count == 10
|
||||
|
||||
_rc.decrypt_stream_chunked(encrypted_path, decrypted_path, key, base_nonce)
|
||||
assert Path(decrypted_path).read_bytes() == data
|
||||
|
||||
def test_invalid_key_length(self, tmp_path):
|
||||
input_path = str(tmp_path / "in")
|
||||
Path(input_path).write_bytes(b"data")
|
||||
|
||||
with pytest.raises(ValueError, match="32 bytes"):
|
||||
_rc.encrypt_stream_chunked(
|
||||
input_path, str(tmp_path / "out"), b"short", secrets.token_bytes(12)
|
||||
)
|
||||
|
||||
def test_invalid_nonce_length(self, tmp_path):
|
||||
input_path = str(tmp_path / "in")
|
||||
Path(input_path).write_bytes(b"data")
|
||||
|
||||
with pytest.raises(ValueError, match="12 bytes"):
|
||||
_rc.encrypt_stream_chunked(
|
||||
input_path, str(tmp_path / "out"), secrets.token_bytes(32), b"short"
|
||||
)
|
||||
|
||||
def test_wrong_key_fails_decrypt(self, tmp_path):
|
||||
data = b"sensitive data"
|
||||
key = secrets.token_bytes(32)
|
||||
wrong_key = secrets.token_bytes(32)
|
||||
base_nonce = secrets.token_bytes(12)
|
||||
|
||||
input_path = str(tmp_path / "plaintext")
|
||||
encrypted_path = str(tmp_path / "encrypted")
|
||||
decrypted_path = str(tmp_path / "decrypted")
|
||||
|
||||
Path(input_path).write_bytes(data)
|
||||
_rc.encrypt_stream_chunked(input_path, encrypted_path, key, base_nonce)
|
||||
|
||||
with pytest.raises((ValueError, OSError)):
|
||||
_rc.decrypt_stream_chunked(
|
||||
encrypted_path, decrypted_path, wrong_key, base_nonce
|
||||
)
|
||||
|
||||
def test_cross_compat_python_encrypt_rust_decrypt(self, tmp_path):
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
data = b"cross compat test data" * 100
|
||||
key = secrets.token_bytes(32)
|
||||
base_nonce = secrets.token_bytes(12)
|
||||
chunk_size = 1024
|
||||
|
||||
encrypted_path = str(tmp_path / "py_encrypted")
|
||||
with open(encrypted_path, "wb") as f:
|
||||
f.write(b"\x00\x00\x00\x00")
|
||||
aesgcm = AESGCM(key)
|
||||
chunk_index = 0
|
||||
offset = 0
|
||||
while offset < len(data):
|
||||
chunk = data[offset:offset + chunk_size]
|
||||
nonce = self._python_derive_chunk_nonce(base_nonce, chunk_index)
|
||||
enc = aesgcm.encrypt(nonce, chunk, None)
|
||||
f.write(len(enc).to_bytes(4, "big"))
|
||||
f.write(enc)
|
||||
chunk_index += 1
|
||||
offset += chunk_size
|
||||
f.seek(0)
|
||||
f.write(chunk_index.to_bytes(4, "big"))
|
||||
|
||||
decrypted_path = str(tmp_path / "rust_decrypted")
|
||||
_rc.decrypt_stream_chunked(encrypted_path, decrypted_path, key, base_nonce)
|
||||
assert Path(decrypted_path).read_bytes() == data
|
||||
|
||||
def test_cross_compat_rust_encrypt_python_decrypt(self, tmp_path):
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
data = b"cross compat reverse test" * 100
|
||||
key = secrets.token_bytes(32)
|
||||
base_nonce = secrets.token_bytes(12)
|
||||
chunk_size = 1024
|
||||
|
||||
input_path = str(tmp_path / "plaintext")
|
||||
encrypted_path = str(tmp_path / "rust_encrypted")
|
||||
Path(input_path).write_bytes(data)
|
||||
|
||||
chunk_count = _rc.encrypt_stream_chunked(
|
||||
input_path, encrypted_path, key, base_nonce, chunk_size=chunk_size
|
||||
)
|
||||
|
||||
aesgcm = AESGCM(key)
|
||||
with open(encrypted_path, "rb") as f:
|
||||
count_bytes = f.read(4)
|
||||
assert int.from_bytes(count_bytes, "big") == chunk_count
|
||||
|
||||
decrypted = b""
|
||||
for i in range(chunk_count):
|
||||
size = int.from_bytes(f.read(4), "big")
|
||||
enc_chunk = f.read(size)
|
||||
nonce = self._python_derive_chunk_nonce(base_nonce, i)
|
||||
decrypted += aesgcm.decrypt(nonce, enc_chunk, None)
|
||||
|
||||
assert decrypted == data
|
||||
|
||||
def test_large_file_roundtrip(self, tmp_path):
|
||||
data = os.urandom(1024 * 1024)
|
||||
key = secrets.token_bytes(32)
|
||||
base_nonce = secrets.token_bytes(12)
|
||||
|
||||
input_path = str(tmp_path / "large")
|
||||
encrypted_path = str(tmp_path / "encrypted")
|
||||
decrypted_path = str(tmp_path / "decrypted")
|
||||
|
||||
Path(input_path).write_bytes(data)
|
||||
|
||||
_rc.encrypt_stream_chunked(input_path, encrypted_path, key, base_nonce)
|
||||
_rc.decrypt_stream_chunked(encrypted_path, decrypted_path, key, base_nonce)
|
||||
|
||||
assert Path(decrypted_path).read_bytes() == data
|
||||
|
||||
|
||||
class TestStreamingEncryptorFileMethods:
|
||||
def test_encrypt_file_decrypt_file_roundtrip(self, tmp_path):
|
||||
from app.encryption import LocalKeyEncryption, StreamingEncryptor
|
||||
|
||||
master_key_path = tmp_path / "master.key"
|
||||
provider = LocalKeyEncryption(master_key_path)
|
||||
encryptor = StreamingEncryptor(provider, chunk_size=512)
|
||||
|
||||
data = b"file method test data" * 200
|
||||
input_path = str(tmp_path / "input")
|
||||
encrypted_path = str(tmp_path / "encrypted")
|
||||
decrypted_path = str(tmp_path / "decrypted")
|
||||
|
||||
Path(input_path).write_bytes(data)
|
||||
|
||||
metadata = encryptor.encrypt_file(input_path, encrypted_path)
|
||||
assert metadata.algorithm == "AES256"
|
||||
|
||||
encryptor.decrypt_file(encrypted_path, decrypted_path, metadata)
|
||||
assert Path(decrypted_path).read_bytes() == data
|
||||
|
||||
def test_encrypt_file_matches_encrypt_stream(self, tmp_path):
|
||||
from app.encryption import LocalKeyEncryption, StreamingEncryptor
|
||||
|
||||
master_key_path = tmp_path / "master.key"
|
||||
provider = LocalKeyEncryption(master_key_path)
|
||||
encryptor = StreamingEncryptor(provider, chunk_size=512)
|
||||
|
||||
data = b"stream vs file comparison" * 100
|
||||
input_path = str(tmp_path / "input")
|
||||
Path(input_path).write_bytes(data)
|
||||
|
||||
file_encrypted_path = str(tmp_path / "file_enc")
|
||||
metadata_file = encryptor.encrypt_file(input_path, file_encrypted_path)
|
||||
|
||||
file_decrypted_path = str(tmp_path / "file_dec")
|
||||
encryptor.decrypt_file(file_encrypted_path, file_decrypted_path, metadata_file)
|
||||
assert Path(file_decrypted_path).read_bytes() == data
|
||||
|
||||
stream_enc, metadata_stream = encryptor.encrypt_stream(io.BytesIO(data))
|
||||
stream_dec = encryptor.decrypt_stream(stream_enc, metadata_stream)
|
||||
assert stream_dec.read() == data
|
||||
@@ -1,8 +1,12 @@
|
||||
import io
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from app import create_app
|
||||
from app.s3_client import S3ProxyClient
|
||||
|
||||
|
||||
def _build_app(tmp_path: Path):
|
||||
@@ -26,13 +30,32 @@ def _build_app(tmp_path: Path):
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://localhost",
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
"SECRET_KEY": "testing",
|
||||
"WTF_CSRF_ENABLED": False,
|
||||
}
|
||||
)
|
||||
|
||||
server = make_server("127.0.0.1", 0, app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
app.config["API_BASE_URL"] = api_url
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
app._test_server = server
|
||||
app._test_thread = thread
|
||||
return app
|
||||
|
||||
|
||||
def _shutdown_app(app):
|
||||
if hasattr(app, "_test_server"):
|
||||
app._test_server.shutdown()
|
||||
app._test_thread.join(timeout=2)
|
||||
|
||||
|
||||
def _login(client):
|
||||
return client.post(
|
||||
"/ui/login",
|
||||
@@ -43,6 +66,7 @@ def _login(client):
|
||||
|
||||
def test_bulk_delete_json_route(tmp_path: Path):
|
||||
app = _build_app(tmp_path)
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("demo")
|
||||
storage.put_object("demo", "first.txt", io.BytesIO(b"first"))
|
||||
@@ -64,10 +88,13 @@ def test_bulk_delete_json_route(tmp_path: Path):
|
||||
|
||||
listing = storage.list_objects_all("demo")
|
||||
assert {meta.key for meta in listing} == {"second.txt"}
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
|
||||
def test_bulk_delete_validation(tmp_path: Path):
|
||||
app = _build_app(tmp_path)
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("demo")
|
||||
storage.put_object("demo", "keep.txt", io.BytesIO(b"keep"))
|
||||
@@ -94,3 +121,5 @@ def test_bulk_delete_validation(tmp_path: Path):
|
||||
|
||||
still_there = storage.list_objects_all("demo")
|
||||
assert {meta.key for meta in still_there} == {"keep.txt"}
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
"""Tests for UI-based encryption configuration."""
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from app import create_app
|
||||
from app.s3_client import S3ProxyClient
|
||||
|
||||
|
||||
def get_csrf_token(response):
|
||||
@@ -43,9 +46,10 @@ def _make_encryption_app(tmp_path: Path, *, kms_enabled: bool = True):
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
"SECRET_KEY": "testing",
|
||||
"ENCRYPTION_ENABLED": True,
|
||||
"WTF_CSRF_ENABLED": False,
|
||||
}
|
||||
|
||||
if kms_enabled:
|
||||
@@ -54,17 +58,37 @@ def _make_encryption_app(tmp_path: Path, *, kms_enabled: bool = True):
|
||||
config["ENCRYPTION_MASTER_KEY_PATH"] = str(tmp_path / "master.key")
|
||||
|
||||
app = create_app(config)
|
||||
|
||||
server = make_server("127.0.0.1", 0, app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
app.config["API_BASE_URL"] = api_url
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
app._test_server = server
|
||||
app._test_thread = thread
|
||||
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
return app
|
||||
|
||||
|
||||
def _shutdown_app(app):
|
||||
if hasattr(app, "_test_server"):
|
||||
app._test_server.shutdown()
|
||||
app._test_thread.join(timeout=2)
|
||||
|
||||
|
||||
class TestUIBucketEncryption:
|
||||
"""Test bucket encryption configuration via UI."""
|
||||
|
||||
def test_bucket_detail_shows_encryption_card(self, tmp_path):
|
||||
"""Encryption card should be visible on bucket detail page."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
@@ -75,21 +99,20 @@ class TestUIBucketEncryption:
|
||||
html = response.data.decode("utf-8")
|
||||
assert "Default Encryption" in html
|
||||
assert "Encryption Algorithm" in html or "Default encryption disabled" in html
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_enable_aes256_encryption(self, tmp_path):
|
||||
"""Should be able to enable AES-256 encryption."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
@@ -99,12 +122,13 @@ class TestUIBucketEncryption:
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "AES-256" in html or "encryption enabled" in html.lower()
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_enable_kms_encryption(self, tmp_path):
|
||||
"""Should be able to enable KMS encryption."""
|
||||
app = _make_encryption_app(tmp_path, kms_enabled=True)
|
||||
client = app.test_client()
|
||||
|
||||
try:
|
||||
with app.app_context():
|
||||
kms = app.extensions.get("kms")
|
||||
if kms:
|
||||
@@ -113,15 +137,12 @@ class TestUIBucketEncryption:
|
||||
else:
|
||||
pytest.skip("KMS not available")
|
||||
|
||||
client = app.test_client()
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "aws:kms",
|
||||
"kms_key_id": key_id,
|
||||
@@ -132,33 +153,28 @@ class TestUIBucketEncryption:
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "KMS" in html or "encryption enabled" in html.lower()
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_disable_encryption(self, tmp_path):
|
||||
"""Should be able to disable encryption."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
|
||||
client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "disable",
|
||||
},
|
||||
follow_redirects=True,
|
||||
@@ -167,21 +183,20 @@ class TestUIBucketEncryption:
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "disabled" in html.lower() or "Default encryption disabled" in html
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_invalid_algorithm_rejected(self, tmp_path):
|
||||
"""Invalid encryption algorithm should be rejected."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "INVALID",
|
||||
},
|
||||
@@ -191,21 +206,20 @@ class TestUIBucketEncryption:
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "Invalid" in html or "danger" in html
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_encryption_persists_in_config(self, tmp_path):
|
||||
"""Encryption config should persist in bucket config."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
|
||||
client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
@@ -217,7 +231,9 @@ class TestUIBucketEncryption:
|
||||
|
||||
assert "Rules" in config
|
||||
assert len(config["Rules"]) == 1
|
||||
assert config["Rules"][0]["ApplyServerSideEncryptionByDefault"]["SSEAlgorithm"] == "AES256"
|
||||
assert config["Rules"][0]["SSEAlgorithm"] == "AES256"
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
|
||||
class TestUIEncryptionWithoutPermission:
|
||||
@@ -226,17 +242,14 @@ class TestUIEncryptionWithoutPermission:
|
||||
def test_readonly_user_cannot_change_encryption(self, tmp_path):
|
||||
"""Read-only user should not be able to change encryption settings."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "readonly", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
@@ -246,3 +259,5 @@ class TestUIEncryptionWithoutPermission:
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "Access denied" in html or "permission" in html.lower() or "not authorized" in html.lower()
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
"""Tests for UI pagination of bucket objects."""
|
||||
import json
|
||||
import threading
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from app import create_app
|
||||
from app.s3_client import S3ProxyClient
|
||||
|
||||
|
||||
def _make_app(tmp_path: Path):
|
||||
"""Create an app for testing."""
|
||||
"""Create an app for testing with a live API server."""
|
||||
storage_root = tmp_path / "data"
|
||||
iam_config = tmp_path / "iam.json"
|
||||
bucket_policies = tmp_path / "bucket_policies.json"
|
||||
@@ -33,29 +36,46 @@ def _make_app(tmp_path: Path):
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
}
|
||||
)
|
||||
|
||||
server = make_server("127.0.0.1", 0, flask_app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
flask_app.config["API_BASE_URL"] = api_url
|
||||
flask_app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
flask_app._test_server = server
|
||||
flask_app._test_thread = thread
|
||||
return flask_app
|
||||
|
||||
|
||||
def _shutdown_app(app):
|
||||
if hasattr(app, "_test_server"):
|
||||
app._test_server.shutdown()
|
||||
app._test_thread.join(timeout=2)
|
||||
|
||||
|
||||
class TestPaginatedObjectListing:
|
||||
"""Test paginated object listing API."""
|
||||
|
||||
def test_objects_api_returns_paginated_results(self, tmp_path):
|
||||
"""Objects API should return paginated results."""
|
||||
app = _make_app(tmp_path)
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
# Create 10 test objects
|
||||
for i in range(10):
|
||||
storage.put_object("test-bucket", f"file{i:02d}.txt", BytesIO(b"content"))
|
||||
|
||||
with app.test_client() as client:
|
||||
# Login first
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
# Request first page of 3 objects
|
||||
resp = client.get("/ui/buckets/test-bucket/objects?max_keys=3")
|
||||
assert resp.status_code == 200
|
||||
|
||||
@@ -63,22 +83,22 @@ class TestPaginatedObjectListing:
|
||||
assert len(data["objects"]) == 3
|
||||
assert data["is_truncated"] is True
|
||||
assert data["next_continuation_token"] is not None
|
||||
assert data["total_count"] == 10
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_objects_api_pagination_continuation(self, tmp_path):
|
||||
"""Objects API should support continuation tokens."""
|
||||
app = _make_app(tmp_path)
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
# Create 5 test objects
|
||||
for i in range(5):
|
||||
storage.put_object("test-bucket", f"file{i:02d}.txt", BytesIO(b"content"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
# Get first page
|
||||
resp = client.get("/ui/buckets/test-bucket/objects?max_keys=2")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
@@ -87,7 +107,6 @@ class TestPaginatedObjectListing:
|
||||
assert len(first_page_keys) == 2
|
||||
assert data["is_truncated"] is True
|
||||
|
||||
# Get second page
|
||||
token = data["next_continuation_token"]
|
||||
resp = client.get(f"/ui/buckets/test-bucket/objects?max_keys=2&continuation_token={token}")
|
||||
assert resp.status_code == 200
|
||||
@@ -96,16 +115,17 @@ class TestPaginatedObjectListing:
|
||||
second_page_keys = [obj["key"] for obj in data["objects"]]
|
||||
assert len(second_page_keys) == 2
|
||||
|
||||
# No overlap between pages
|
||||
assert set(first_page_keys).isdisjoint(set(second_page_keys))
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_objects_api_prefix_filter(self, tmp_path):
|
||||
"""Objects API should support prefix filtering."""
|
||||
app = _make_app(tmp_path)
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
# Create objects with different prefixes
|
||||
storage.put_object("test-bucket", "logs/access.log", BytesIO(b"log"))
|
||||
storage.put_object("test-bucket", "logs/error.log", BytesIO(b"log"))
|
||||
storage.put_object("test-bucket", "data/file.txt", BytesIO(b"data"))
|
||||
@@ -113,7 +133,6 @@ class TestPaginatedObjectListing:
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
# Filter by prefix
|
||||
resp = client.get("/ui/buckets/test-bucket/objects?prefix=logs/")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
@@ -121,23 +140,27 @@ class TestPaginatedObjectListing:
|
||||
keys = [obj["key"] for obj in data["objects"]]
|
||||
assert all(k.startswith("logs/") for k in keys)
|
||||
assert len(keys) == 2
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_objects_api_requires_authentication(self, tmp_path):
|
||||
"""Objects API should require login."""
|
||||
app = _make_app(tmp_path)
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
with app.test_client() as client:
|
||||
# Don't login
|
||||
resp = client.get("/ui/buckets/test-bucket/objects")
|
||||
# Should redirect to login
|
||||
assert resp.status_code == 302
|
||||
assert "/ui/login" in resp.headers.get("Location", "")
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_objects_api_returns_object_metadata(self, tmp_path):
|
||||
"""Objects API should return complete object metadata."""
|
||||
app = _make_app(tmp_path)
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
storage.put_object("test-bucket", "test.txt", BytesIO(b"test content"))
|
||||
@@ -152,38 +175,38 @@ class TestPaginatedObjectListing:
|
||||
assert len(data["objects"]) == 1
|
||||
obj = data["objects"][0]
|
||||
|
||||
# Check all expected fields
|
||||
assert obj["key"] == "test.txt"
|
||||
assert obj["size"] == 12 # len("test content")
|
||||
assert obj["size"] == 12
|
||||
assert "last_modified" in obj
|
||||
assert "last_modified_display" in obj
|
||||
assert "etag" in obj
|
||||
|
||||
# URLs are now returned as templates (not per-object) for performance
|
||||
assert "url_templates" in data
|
||||
templates = data["url_templates"]
|
||||
assert "preview" in templates
|
||||
assert "download" in templates
|
||||
assert "delete" in templates
|
||||
assert "KEY_PLACEHOLDER" in templates["preview"]
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_bucket_detail_page_loads_without_objects(self, tmp_path):
|
||||
"""Bucket detail page should load even with many objects."""
|
||||
app = _make_app(tmp_path)
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
# Create many objects
|
||||
for i in range(100):
|
||||
storage.put_object("test-bucket", f"file{i:03d}.txt", BytesIO(b"x"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
# The page should load quickly (objects loaded via JS)
|
||||
resp = client.get("/ui/buckets/test-bucket")
|
||||
assert resp.status_code == 200
|
||||
|
||||
html = resp.data.decode("utf-8")
|
||||
# Should have the JavaScript loading infrastructure (external JS file)
|
||||
assert "bucket-detail-main.js" in html
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import io
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from app import create_app
|
||||
from app.s3_client import S3ProxyClient
|
||||
|
||||
|
||||
DENY_LIST_ALLOW_GET_POLICY = {
|
||||
@@ -47,11 +50,25 @@ def _make_ui_app(tmp_path: Path, *, enforce_policies: bool):
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
"SECRET_KEY": "testing",
|
||||
"UI_ENFORCE_BUCKET_POLICIES": enforce_policies,
|
||||
"WTF_CSRF_ENABLED": False,
|
||||
}
|
||||
)
|
||||
|
||||
server = make_server("127.0.0.1", 0, app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
app.config["API_BASE_URL"] = api_url
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
app._test_server = server
|
||||
app._test_thread = thread
|
||||
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("testbucket")
|
||||
storage.put_object("testbucket", "vid.mp4", io.BytesIO(b"video"))
|
||||
@@ -60,9 +77,16 @@ def _make_ui_app(tmp_path: Path, *, enforce_policies: bool):
|
||||
return app
|
||||
|
||||
|
||||
def _shutdown_app(app):
|
||||
if hasattr(app, "_test_server"):
|
||||
app._test_server.shutdown()
|
||||
app._test_thread.join(timeout=2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enforce", [True, False])
|
||||
def test_ui_bucket_policy_enforcement_toggle(tmp_path: Path, enforce: bool):
|
||||
app = _make_ui_app(tmp_path, enforce_policies=enforce)
|
||||
try:
|
||||
client = app.test_client()
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
response = client.get("/ui/buckets/testbucket", follow_redirects=True)
|
||||
@@ -71,11 +95,10 @@ def test_ui_bucket_policy_enforcement_toggle(tmp_path: Path, enforce: bool):
|
||||
else:
|
||||
assert response.status_code == 200
|
||||
assert b"Access denied by bucket policy" not in response.data
|
||||
# Objects are now loaded via async API - check the objects endpoint
|
||||
objects_response = client.get("/ui/buckets/testbucket/objects")
|
||||
assert objects_response.status_code == 200
|
||||
data = objects_response.get_json()
|
||||
assert any(obj["key"] == "vid.mp4" for obj in data["objects"])
|
||||
assert objects_response.status_code == 403
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
|
||||
def test_ui_bucket_policy_disabled_by_default(tmp_path: Path):
|
||||
@@ -99,10 +122,25 @@ def test_ui_bucket_policy_disabled_by_default(tmp_path: Path):
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
"SECRET_KEY": "testing",
|
||||
"WTF_CSRF_ENABLED": False,
|
||||
}
|
||||
)
|
||||
|
||||
server = make_server("127.0.0.1", 0, app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
app.config["API_BASE_URL"] = api_url
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
app._test_server = server
|
||||
app._test_thread = thread
|
||||
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("testbucket")
|
||||
storage.put_object("testbucket", "vid.mp4", io.BytesIO(b"video"))
|
||||
@@ -114,8 +152,7 @@ def test_ui_bucket_policy_disabled_by_default(tmp_path: Path):
|
||||
response = client.get("/ui/buckets/testbucket", follow_redirects=True)
|
||||
assert response.status_code == 200
|
||||
assert b"Access denied by bucket policy" not in response.data
|
||||
# Objects are now loaded via async API - check the objects endpoint
|
||||
objects_response = client.get("/ui/buckets/testbucket/objects")
|
||||
assert objects_response.status_code == 200
|
||||
data = objects_response.get_json()
|
||||
assert any(obj["key"] == "vid.mp4" for obj in data["objects"])
|
||||
assert objects_response.status_code == 403
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
442
tests/test_website_hosting.py
Normal file
442
tests/test_website_hosting.py
Normal file
@@ -0,0 +1,442 @@
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
from xml.etree.ElementTree import fromstring
|
||||
|
||||
import pytest
|
||||
|
||||
from app import create_api_app
|
||||
from app.website_domains import WebsiteDomainStore
|
||||
|
||||
|
||||
def _stream(data: bytes):
|
||||
return io.BytesIO(data)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def website_app(tmp_path: Path):
|
||||
storage_root = tmp_path / "data"
|
||||
iam_config = tmp_path / "iam.json"
|
||||
bucket_policies = tmp_path / "bucket_policies.json"
|
||||
iam_payload = {
|
||||
"users": [
|
||||
{
|
||||
"access_key": "test",
|
||||
"secret_key": "secret",
|
||||
"display_name": "Test User",
|
||||
"policies": [{"bucket": "*", "actions": ["list", "read", "write", "delete", "policy", "iam:*"]}],
|
||||
}
|
||||
]
|
||||
}
|
||||
iam_config.write_text(json.dumps(iam_payload))
|
||||
flask_app = create_api_app(
|
||||
{
|
||||
"TESTING": True,
|
||||
"SECRET_KEY": "testing",
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"WEBSITE_HOSTING_ENABLED": True,
|
||||
}
|
||||
)
|
||||
yield flask_app
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def website_client(website_app):
|
||||
return website_app.test_client()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def storage(website_app):
|
||||
return website_app.extensions["object_storage"]
|
||||
|
||||
|
||||
class TestWebsiteDomainStore:
|
||||
def test_empty_store(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
assert store.list_all() == []
|
||||
assert store.get_bucket("example.com") is None
|
||||
|
||||
def test_set_and_get_mapping(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("example.com", "my-site")
|
||||
assert store.get_bucket("example.com") == "my-site"
|
||||
|
||||
def test_case_insensitive(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("Example.COM", "my-site")
|
||||
assert store.get_bucket("example.com") == "my-site"
|
||||
assert store.get_bucket("EXAMPLE.COM") == "my-site"
|
||||
|
||||
def test_list_all(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("a.com", "bucket-a")
|
||||
store.set_mapping("b.com", "bucket-b")
|
||||
result = store.list_all()
|
||||
domains = {item["domain"] for item in result}
|
||||
assert domains == {"a.com", "b.com"}
|
||||
|
||||
def test_delete_mapping(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("example.com", "my-site")
|
||||
assert store.delete_mapping("example.com") is True
|
||||
assert store.get_bucket("example.com") is None
|
||||
|
||||
def test_delete_nonexistent(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
assert store.delete_mapping("nope.com") is False
|
||||
|
||||
def test_overwrite_mapping(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("example.com", "old-bucket")
|
||||
store.set_mapping("example.com", "new-bucket")
|
||||
assert store.get_bucket("example.com") == "new-bucket"
|
||||
|
||||
def test_persistence(self, tmp_path):
|
||||
path = tmp_path / "domains.json"
|
||||
store1 = WebsiteDomainStore(path)
|
||||
store1.set_mapping("example.com", "my-site")
|
||||
store2 = WebsiteDomainStore(path)
|
||||
assert store2.get_bucket("example.com") == "my-site"
|
||||
|
||||
def test_corrupt_file(self, tmp_path):
|
||||
path = tmp_path / "domains.json"
|
||||
path.write_text("not json")
|
||||
store = WebsiteDomainStore(path)
|
||||
assert store.list_all() == []
|
||||
|
||||
def test_non_dict_file(self, tmp_path):
|
||||
path = tmp_path / "domains.json"
|
||||
path.write_text('["not", "a", "dict"]')
|
||||
store = WebsiteDomainStore(path)
|
||||
assert store.list_all() == []
|
||||
|
||||
|
||||
class TestStorageWebsiteConfig:
|
||||
def test_get_website_no_config(self, storage):
|
||||
storage.create_bucket("test-bucket")
|
||||
assert storage.get_bucket_website("test-bucket") is None
|
||||
|
||||
def test_set_and_get_website(self, storage):
|
||||
storage.create_bucket("test-bucket")
|
||||
config = {"index_document": "index.html", "error_document": "error.html"}
|
||||
storage.set_bucket_website("test-bucket", config)
|
||||
result = storage.get_bucket_website("test-bucket")
|
||||
assert result["index_document"] == "index.html"
|
||||
assert result["error_document"] == "error.html"
|
||||
|
||||
def test_delete_website_config(self, storage):
|
||||
storage.create_bucket("test-bucket")
|
||||
storage.set_bucket_website("test-bucket", {"index_document": "index.html"})
|
||||
storage.set_bucket_website("test-bucket", None)
|
||||
assert storage.get_bucket_website("test-bucket") is None
|
||||
|
||||
def test_nonexistent_bucket(self, storage):
|
||||
with pytest.raises(Exception):
|
||||
storage.get_bucket_website("no-such-bucket")
|
||||
|
||||
|
||||
class TestS3WebsiteAPI:
|
||||
def test_put_website_config(self, website_client, signer):
|
||||
headers = signer("PUT", "/site-bucket")
|
||||
assert website_client.put("/site-bucket", headers=headers).status_code == 200
|
||||
|
||||
xml_body = b"""<WebsiteConfiguration>
|
||||
<IndexDocument><Suffix>index.html</Suffix></IndexDocument>
|
||||
<ErrorDocument><Key>404.html</Key></ErrorDocument>
|
||||
</WebsiteConfiguration>"""
|
||||
headers = signer("PUT", "/site-bucket?website",
|
||||
headers={"Content-Type": "application/xml"}, body=xml_body)
|
||||
resp = website_client.put("/site-bucket", query_string={"website": ""},
|
||||
headers=headers, data=xml_body, content_type="application/xml")
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_get_website_config(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
storage.set_bucket_website("site-bucket", {
|
||||
"index_document": "index.html",
|
||||
"error_document": "error.html",
|
||||
})
|
||||
|
||||
headers = signer("GET", "/site-bucket?website")
|
||||
resp = website_client.get("/site-bucket", query_string={"website": ""}, headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
root = fromstring(resp.data)
|
||||
suffix = root.find(".//{http://s3.amazonaws.com/doc/2006-03-01/}Suffix")
|
||||
if suffix is None:
|
||||
suffix = root.find(".//Suffix")
|
||||
assert suffix is not None
|
||||
assert suffix.text == "index.html"
|
||||
|
||||
def test_get_website_config_not_set(self, website_client, signer, storage):
|
||||
storage.create_bucket("no-website")
|
||||
headers = signer("GET", "/no-website?website")
|
||||
resp = website_client.get("/no-website", query_string={"website": ""}, headers=headers)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_delete_website_config(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
storage.set_bucket_website("site-bucket", {"index_document": "index.html"})
|
||||
|
||||
headers = signer("DELETE", "/site-bucket?website")
|
||||
resp = website_client.delete("/site-bucket", query_string={"website": ""}, headers=headers)
|
||||
assert resp.status_code == 204
|
||||
assert storage.get_bucket_website("site-bucket") is None
|
||||
|
||||
def test_put_website_missing_index(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
xml_body = b"""<WebsiteConfiguration>
|
||||
<ErrorDocument><Key>error.html</Key></ErrorDocument>
|
||||
</WebsiteConfiguration>"""
|
||||
headers = signer("PUT", "/site-bucket?website",
|
||||
headers={"Content-Type": "application/xml"}, body=xml_body)
|
||||
resp = website_client.put("/site-bucket", query_string={"website": ""},
|
||||
headers=headers, data=xml_body, content_type="application/xml")
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_put_website_slash_in_suffix(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
xml_body = b"""<WebsiteConfiguration>
|
||||
<IndexDocument><Suffix>path/index.html</Suffix></IndexDocument>
|
||||
</WebsiteConfiguration>"""
|
||||
headers = signer("PUT", "/site-bucket?website",
|
||||
headers={"Content-Type": "application/xml"}, body=xml_body)
|
||||
resp = website_client.put("/site-bucket", query_string={"website": ""},
|
||||
headers=headers, data=xml_body, content_type="application/xml")
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_put_website_malformed_xml(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
xml_body = b"not xml at all"
|
||||
headers = signer("PUT", "/site-bucket?website",
|
||||
headers={"Content-Type": "application/xml"}, body=xml_body)
|
||||
resp = website_client.put("/site-bucket", query_string={"website": ""},
|
||||
headers=headers, data=xml_body, content_type="application/xml")
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_website_disabled(self, client, signer):
|
||||
headers = signer("PUT", "/test-bucket")
|
||||
assert client.put("/test-bucket", headers=headers).status_code == 200
|
||||
headers = signer("GET", "/test-bucket?website")
|
||||
resp = client.get("/test-bucket", query_string={"website": ""}, headers=headers)
|
||||
assert resp.status_code == 400
|
||||
assert b"not enabled" in resp.data
|
||||
|
||||
|
||||
class TestAdminWebsiteDomains:
|
||||
def _admin_headers(self, signer):
|
||||
return signer("GET", "/admin/website-domains")
|
||||
|
||||
def test_list_empty(self, website_client, signer):
|
||||
headers = self._admin_headers(signer)
|
||||
resp = website_client.get("/admin/website-domains", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == []
|
||||
|
||||
def test_create_mapping(self, website_client, signer, storage):
|
||||
storage.create_bucket("my-site")
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"},
|
||||
body=json.dumps({"domain": "example.com", "bucket": "my-site"}).encode())
|
||||
resp = website_client.post("/admin/website-domains",
|
||||
headers=headers,
|
||||
json={"domain": "example.com", "bucket": "my-site"})
|
||||
assert resp.status_code == 201
|
||||
data = resp.get_json()
|
||||
assert data["domain"] == "example.com"
|
||||
assert data["bucket"] == "my-site"
|
||||
|
||||
def test_create_duplicate(self, website_client, signer, storage):
|
||||
storage.create_bucket("my-site")
|
||||
body = json.dumps({"domain": "dup.com", "bucket": "my-site"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "dup.com", "bucket": "my-site"})
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
resp = website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "dup.com", "bucket": "my-site"})
|
||||
assert resp.status_code == 409
|
||||
|
||||
def test_create_missing_domain(self, website_client, signer, storage):
|
||||
storage.create_bucket("my-site")
|
||||
body = json.dumps({"bucket": "my-site"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
resp = website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"bucket": "my-site"})
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_create_nonexistent_bucket(self, website_client, signer):
|
||||
body = json.dumps({"domain": "x.com", "bucket": "no-such"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
resp = website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "x.com", "bucket": "no-such"})
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_get_mapping(self, website_client, signer, storage):
|
||||
storage.create_bucket("my-site")
|
||||
body = json.dumps({"domain": "get.com", "bucket": "my-site"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "get.com", "bucket": "my-site"})
|
||||
|
||||
headers = signer("GET", "/admin/website-domains/get.com")
|
||||
resp = website_client.get("/admin/website-domains/get.com", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["bucket"] == "my-site"
|
||||
|
||||
def test_get_nonexistent(self, website_client, signer):
|
||||
headers = signer("GET", "/admin/website-domains/nope.com")
|
||||
resp = website_client.get("/admin/website-domains/nope.com", headers=headers)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_update_mapping(self, website_client, signer, storage):
|
||||
storage.create_bucket("old-bucket")
|
||||
storage.create_bucket("new-bucket")
|
||||
body = json.dumps({"domain": "upd.com", "bucket": "old-bucket"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "upd.com", "bucket": "old-bucket"})
|
||||
|
||||
body = json.dumps({"bucket": "new-bucket"}).encode()
|
||||
headers = signer("PUT", "/admin/website-domains/upd.com",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
resp = website_client.put("/admin/website-domains/upd.com", headers=headers,
|
||||
json={"bucket": "new-bucket"})
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["bucket"] == "new-bucket"
|
||||
|
||||
def test_delete_mapping(self, website_client, signer, storage):
|
||||
storage.create_bucket("del-bucket")
|
||||
body = json.dumps({"domain": "del.com", "bucket": "del-bucket"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "del.com", "bucket": "del-bucket"})
|
||||
|
||||
headers = signer("DELETE", "/admin/website-domains/del.com")
|
||||
resp = website_client.delete("/admin/website-domains/del.com", headers=headers)
|
||||
assert resp.status_code == 204
|
||||
|
||||
def test_delete_nonexistent(self, website_client, signer):
|
||||
headers = signer("DELETE", "/admin/website-domains/nope.com")
|
||||
resp = website_client.delete("/admin/website-domains/nope.com", headers=headers)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_disabled(self, website_client, signer):
|
||||
with website_client.application.test_request_context():
|
||||
website_client.application.config["WEBSITE_HOSTING_ENABLED"] = False
|
||||
headers = signer("GET", "/admin/website-domains")
|
||||
resp = website_client.get("/admin/website-domains", headers=headers)
|
||||
assert resp.status_code == 400
|
||||
website_client.application.config["WEBSITE_HOSTING_ENABLED"] = True
|
||||
|
||||
|
||||
class TestWebsiteServing:
|
||||
def _setup_website(self, storage, website_app):
|
||||
storage.create_bucket("my-site")
|
||||
storage.put_object("my-site", "index.html", _stream(b"<h1>Home</h1>"))
|
||||
storage.put_object("my-site", "about.html", _stream(b"<h1>About</h1>"))
|
||||
storage.put_object("my-site", "assets/style.css", _stream(b"body { color: red; }"))
|
||||
storage.put_object("my-site", "sub/index.html", _stream(b"<h1>Sub</h1>"))
|
||||
storage.put_object("my-site", "404.html", _stream(b"<h1>Not Found</h1>"))
|
||||
storage.set_bucket_website("my-site", {
|
||||
"index_document": "index.html",
|
||||
"error_document": "404.html",
|
||||
})
|
||||
store = website_app.extensions["website_domains"]
|
||||
store.set_mapping("mysite.example.com", "my-site")
|
||||
|
||||
def test_serve_index(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>Home</h1>" in resp.data
|
||||
assert "text/html" in resp.content_type
|
||||
|
||||
def test_serve_specific_file(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/about.html", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>About</h1>" in resp.data
|
||||
|
||||
def test_serve_css(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/assets/style.css", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"body { color: red; }" in resp.data
|
||||
assert "text/css" in resp.content_type
|
||||
|
||||
def test_serve_subdirectory_index(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/sub/", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>Sub</h1>" in resp.data
|
||||
|
||||
def test_serve_subdirectory_no_trailing_slash(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/sub", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>Sub</h1>" in resp.data
|
||||
|
||||
def test_serve_error_document(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/nonexistent.html", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 404
|
||||
assert b"<h1>Not Found</h1>" in resp.data
|
||||
|
||||
def test_unmapped_host_passes_through(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/", headers={"Host": "unknown.example.com"})
|
||||
assert resp.status_code != 200 or b"<h1>Home</h1>" not in resp.data
|
||||
|
||||
def test_head_request(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.head("/index.html", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert "Content-Length" in resp.headers
|
||||
assert resp.data == b""
|
||||
|
||||
def test_post_not_intercepted(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.post("/index.html", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code != 200 or b"<h1>Home</h1>" not in resp.data
|
||||
|
||||
def test_bucket_deleted(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
for obj in storage.list_objects_all("my-site"):
|
||||
storage.delete_object("my-site", obj.key)
|
||||
storage.delete_bucket("my-site")
|
||||
resp = website_client.get("/", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_no_website_config(self, website_client, storage, website_app):
|
||||
storage.create_bucket("bare-bucket")
|
||||
store = website_app.extensions["website_domains"]
|
||||
store.set_mapping("bare.example.com", "bare-bucket")
|
||||
resp = website_client.get("/", headers={"Host": "bare.example.com"})
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_host_with_port(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/", headers={"Host": "mysite.example.com:5000"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>Home</h1>" in resp.data
|
||||
|
||||
def test_no_error_document(self, website_client, storage, website_app):
|
||||
storage.create_bucket("no-err")
|
||||
storage.put_object("no-err", "index.html", _stream(b"<h1>Home</h1>"))
|
||||
storage.set_bucket_website("no-err", {"index_document": "index.html"})
|
||||
store = website_app.extensions["website_domains"]
|
||||
store.set_mapping("noerr.example.com", "no-err")
|
||||
resp = website_client.get("/missing.html", headers={"Host": "noerr.example.com"})
|
||||
assert resp.status_code == 404
|
||||
assert b"Not Found" in resp.data
|
||||
Reference in New Issue
Block a user