Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d96955deee | |||
| 77a46d0725 | |||
| 0f750b9d89 | |||
| e0dee9db36 | |||
| 126657c99f | |||
| 07fb1ac773 | |||
| 147962e1dd | |||
| 2643a79121 | |||
| e9a035827b | |||
| 033b8a82be | |||
| e76c311231 | |||
| cbdf1a27c8 | |||
| 4a60cb269a | |||
| ebe7f6222d | |||
| 70b61fd8e6 | |||
| 85181f0be6 | |||
| a779b002d7 |
@@ -1,12 +1,10 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
FROM python:3.12.12-slim
|
||||
FROM python:3.14.3-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install build deps for any wheels that need compilation, then clean up
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
@@ -16,10 +14,8 @@ RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
# Make entrypoint executable
|
||||
RUN chmod +x docker-entrypoint.sh
|
||||
|
||||
# Create data directory and set permissions
|
||||
RUN mkdir -p /app/data \
|
||||
&& useradd -m -u 1000 myfsio \
|
||||
&& chown -R myfsio:myfsio /app
|
||||
|
||||
@@ -263,11 +263,37 @@ def create_app(
|
||||
|
||||
@app.errorhandler(500)
|
||||
def internal_error(error):
|
||||
return render_template('500.html'), 500
|
||||
wants_html = request.accept_mimetypes.accept_html
|
||||
path = request.path or ""
|
||||
if include_ui and wants_html and (path.startswith("/ui") or path == "/"):
|
||||
return render_template('500.html'), 500
|
||||
error_xml = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
'<Error>'
|
||||
'<Code>InternalError</Code>'
|
||||
'<Message>An internal server error occurred</Message>'
|
||||
f'<Resource>{path}</Resource>'
|
||||
f'<RequestId>{getattr(g, "request_id", "-")}</RequestId>'
|
||||
'</Error>'
|
||||
)
|
||||
return error_xml, 500, {'Content-Type': 'application/xml'}
|
||||
|
||||
@app.errorhandler(CSRFError)
|
||||
def handle_csrf_error(e):
|
||||
return render_template('csrf_error.html', reason=e.description), 400
|
||||
wants_html = request.accept_mimetypes.accept_html
|
||||
path = request.path or ""
|
||||
if include_ui and wants_html and (path.startswith("/ui") or path == "/"):
|
||||
return render_template('csrf_error.html', reason=e.description), 400
|
||||
error_xml = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
'<Error>'
|
||||
'<Code>CSRFError</Code>'
|
||||
f'<Message>{e.description}</Message>'
|
||||
f'<Resource>{path}</Resource>'
|
||||
f'<RequestId>{getattr(g, "request_id", "-")}</RequestId>'
|
||||
'</Error>'
|
||||
)
|
||||
return error_xml, 400, {'Content-Type': 'application/xml'}
|
||||
|
||||
@app.template_filter("filesizeformat")
|
||||
def filesizeformat(value: int) -> str:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
@@ -354,6 +355,10 @@ def update_peer_site(site_id: str):
|
||||
if region_error:
|
||||
return _json_error("ValidationError", region_error, 400)
|
||||
|
||||
if "connection_id" in payload:
|
||||
if payload["connection_id"] and not _connections().get(payload["connection_id"]):
|
||||
return _json_error("ValidationError", f"Connection '{payload['connection_id']}' not found", 400)
|
||||
|
||||
peer = PeerSite(
|
||||
site_id=site_id,
|
||||
endpoint=payload.get("endpoint", existing.endpoint),
|
||||
|
||||
@@ -6,6 +6,7 @@ import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from fnmatch import fnmatch, translate
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Pattern, Sequence, Tuple
|
||||
|
||||
@@ -13,9 +14,14 @@ from typing import Any, Dict, Iterable, List, Optional, Pattern, Sequence, Tuple
|
||||
RESOURCE_PREFIX = "arn:aws:s3:::"
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _compile_pattern(pattern: str) -> Pattern[str]:
|
||||
return re.compile(translate(pattern), re.IGNORECASE)
|
||||
|
||||
|
||||
def _match_string_like(value: str, pattern: str) -> bool:
|
||||
regex = translate(pattern)
|
||||
return bool(re.match(regex, value, re.IGNORECASE))
|
||||
compiled = _compile_pattern(pattern)
|
||||
return bool(compiled.match(value))
|
||||
|
||||
|
||||
def _ip_in_cidr(ip_str: str, cidr: str) -> bool:
|
||||
|
||||
@@ -36,10 +36,11 @@ class GzipMiddleware:
|
||||
content_type = None
|
||||
content_length = None
|
||||
should_compress = False
|
||||
is_streaming = False
|
||||
exc_info_holder = [None]
|
||||
|
||||
def custom_start_response(status: str, headers: List[Tuple[str, str]], exc_info=None):
|
||||
nonlocal response_started, status_code, response_headers, content_type, content_length, should_compress
|
||||
nonlocal response_started, status_code, response_headers, content_type, content_length, should_compress, is_streaming
|
||||
response_started = True
|
||||
status_code = int(status.split(' ', 1)[0])
|
||||
response_headers = list(headers)
|
||||
@@ -54,6 +55,9 @@ class GzipMiddleware:
|
||||
elif name_lower == 'content-encoding':
|
||||
should_compress = False
|
||||
return start_response(status, headers, exc_info)
|
||||
elif name_lower == 'x-stream-response':
|
||||
is_streaming = True
|
||||
return start_response(status, headers, exc_info)
|
||||
|
||||
if content_type and content_type in COMPRESSIBLE_MIMES:
|
||||
if content_length is None or content_length >= self.min_size:
|
||||
@@ -61,7 +65,12 @@ class GzipMiddleware:
|
||||
|
||||
return None
|
||||
|
||||
response_body = b''.join(self.app(environ, custom_start_response))
|
||||
app_iter = self.app(environ, custom_start_response)
|
||||
|
||||
if is_streaming:
|
||||
return app_iter
|
||||
|
||||
response_body = b''.join(app_iter)
|
||||
|
||||
if not response_started:
|
||||
return [response_body]
|
||||
|
||||
37
app/iam.py
37
app/iam.py
@@ -4,6 +4,7 @@ import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
@@ -121,7 +122,8 @@ class IamService:
|
||||
self._failed_attempts: Dict[str, Deque[datetime]] = {}
|
||||
self._last_load_time = 0.0
|
||||
self._principal_cache: Dict[str, Tuple[Principal, float]] = {}
|
||||
self._cache_ttl = 10.0
|
||||
self._secret_key_cache: Dict[str, Tuple[str, float]] = {}
|
||||
self._cache_ttl = float(os.environ.get("IAM_CACHE_TTL_SECONDS", "5.0"))
|
||||
self._last_stat_check = 0.0
|
||||
self._stat_check_interval = 1.0
|
||||
self._sessions: Dict[str, Dict[str, Any]] = {}
|
||||
@@ -139,6 +141,7 @@ class IamService:
|
||||
if self.config_path.stat().st_mtime > self._last_load_time:
|
||||
self._load()
|
||||
self._principal_cache.clear()
|
||||
self._secret_key_cache.clear()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@@ -367,6 +370,9 @@ class IamService:
|
||||
user["secret_key"] = new_secret
|
||||
self._save()
|
||||
self._principal_cache.pop(access_key, None)
|
||||
self._secret_key_cache.pop(access_key, None)
|
||||
from .s3_api import clear_signing_key_cache
|
||||
clear_signing_key_cache()
|
||||
self._load()
|
||||
return new_secret
|
||||
|
||||
@@ -385,6 +391,10 @@ class IamService:
|
||||
raise IamError("User not found")
|
||||
self._raw_config["users"] = remaining
|
||||
self._save()
|
||||
self._principal_cache.pop(access_key, None)
|
||||
self._secret_key_cache.pop(access_key, None)
|
||||
from .s3_api import clear_signing_key_cache
|
||||
clear_signing_key_cache()
|
||||
self._load()
|
||||
|
||||
def update_user_policies(self, access_key: str, policies: Sequence[Dict[str, Any]]) -> None:
|
||||
@@ -519,11 +529,13 @@ class IamService:
|
||||
return candidate if candidate in ALLOWED_ACTIONS else ""
|
||||
|
||||
def _write_default(self) -> None:
|
||||
access_key = secrets.token_hex(12)
|
||||
secret_key = secrets.token_urlsafe(32)
|
||||
default = {
|
||||
"users": [
|
||||
{
|
||||
"access_key": "localadmin",
|
||||
"secret_key": "localadmin",
|
||||
"access_key": access_key,
|
||||
"secret_key": secret_key,
|
||||
"display_name": "Local Admin",
|
||||
"policies": [
|
||||
{"bucket": "*", "actions": list(ALLOWED_ACTIONS)}
|
||||
@@ -532,6 +544,14 @@ class IamService:
|
||||
]
|
||||
}
|
||||
self.config_path.write_text(json.dumps(default, indent=2))
|
||||
print(f"\n{'='*60}")
|
||||
print("MYFSIO FIRST RUN - ADMIN CREDENTIALS GENERATED")
|
||||
print(f"{'='*60}")
|
||||
print(f"Access Key: {access_key}")
|
||||
print(f"Secret Key: {secret_key}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Missed this? Check: {self.config_path}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
def _generate_access_key(self) -> str:
|
||||
return secrets.token_hex(8)
|
||||
@@ -546,10 +566,19 @@ class IamService:
|
||||
raise IamError("User not found")
|
||||
|
||||
def get_secret_key(self, access_key: str) -> str | None:
|
||||
now = time.time()
|
||||
cached = self._secret_key_cache.get(access_key)
|
||||
if cached:
|
||||
secret_key, cached_time = cached
|
||||
if now - cached_time < self._cache_ttl:
|
||||
return secret_key
|
||||
|
||||
self._maybe_reload()
|
||||
record = self._users.get(access_key)
|
||||
if record:
|
||||
return record["secret_key"]
|
||||
secret_key = record["secret_key"]
|
||||
self._secret_key_cache[access_key] = (secret_key, now)
|
||||
return secret_key
|
||||
return None
|
||||
|
||||
def get_principal(self, access_key: str) -> Principal | None:
|
||||
|
||||
@@ -6,9 +6,12 @@ import hmac
|
||||
import logging
|
||||
import mimetypes
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from urllib.parse import quote, urlencode, urlparse, unquote
|
||||
from xml.etree.ElementTree import Element, SubElement, tostring, ParseError
|
||||
from defusedxml.ElementTree import fromstring
|
||||
@@ -181,11 +184,41 @@ def _sign(key: bytes, msg: str) -> bytes:
|
||||
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
|
||||
|
||||
|
||||
_SIGNING_KEY_CACHE: OrderedDict[Tuple[str, str, str, str], Tuple[bytes, float]] = OrderedDict()
|
||||
_SIGNING_KEY_CACHE_LOCK = threading.Lock()
|
||||
_SIGNING_KEY_CACHE_TTL = 60.0
|
||||
_SIGNING_KEY_CACHE_MAX_SIZE = 256
|
||||
|
||||
|
||||
def clear_signing_key_cache() -> None:
|
||||
with _SIGNING_KEY_CACHE_LOCK:
|
||||
_SIGNING_KEY_CACHE.clear()
|
||||
|
||||
|
||||
def _get_signature_key(key: str, date_stamp: str, region_name: str, service_name: str) -> bytes:
|
||||
cache_key = (key, date_stamp, region_name, service_name)
|
||||
now = time.time()
|
||||
|
||||
with _SIGNING_KEY_CACHE_LOCK:
|
||||
cached = _SIGNING_KEY_CACHE.get(cache_key)
|
||||
if cached:
|
||||
signing_key, cached_time = cached
|
||||
if now - cached_time < _SIGNING_KEY_CACHE_TTL:
|
||||
_SIGNING_KEY_CACHE.move_to_end(cache_key)
|
||||
return signing_key
|
||||
else:
|
||||
del _SIGNING_KEY_CACHE[cache_key]
|
||||
|
||||
k_date = _sign(("AWS4" + key).encode("utf-8"), date_stamp)
|
||||
k_region = _sign(k_date, region_name)
|
||||
k_service = _sign(k_region, service_name)
|
||||
k_signing = _sign(k_service, "aws4_request")
|
||||
|
||||
with _SIGNING_KEY_CACHE_LOCK:
|
||||
if len(_SIGNING_KEY_CACHE) >= _SIGNING_KEY_CACHE_MAX_SIZE:
|
||||
_SIGNING_KEY_CACHE.popitem(last=False)
|
||||
_SIGNING_KEY_CACHE[cache_key] = (k_signing, now)
|
||||
|
||||
return k_signing
|
||||
|
||||
|
||||
@@ -971,7 +1004,8 @@ def _apply_object_headers(
|
||||
response.headers["ETag"] = f'"{etag}"'
|
||||
response.headers["Accept-Ranges"] = "bytes"
|
||||
for key, value in (metadata or {}).items():
|
||||
response.headers[f"X-Amz-Meta-{key}"] = value
|
||||
safe_value = _sanitize_header_value(str(value))
|
||||
response.headers[f"X-Amz-Meta-{key}"] = safe_value
|
||||
|
||||
|
||||
def _maybe_handle_bucket_subresource(bucket_name: str) -> Response | None:
|
||||
@@ -2309,10 +2343,12 @@ def _post_object(bucket_name: str) -> Response:
|
||||
success_action_redirect = request.form.get("success_action_redirect")
|
||||
if success_action_redirect:
|
||||
allowed_hosts = current_app.config.get("ALLOWED_REDIRECT_HOSTS", [])
|
||||
if not allowed_hosts:
|
||||
allowed_hosts = [request.host]
|
||||
parsed = urlparse(success_action_redirect)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
return _error_response("InvalidArgument", "Redirect URL must use http or https", 400)
|
||||
if allowed_hosts and parsed.netloc not in allowed_hosts:
|
||||
if parsed.netloc not in allowed_hosts:
|
||||
return _error_response("InvalidArgument", "Redirect URL host not allowed", 400)
|
||||
redirect_url = f"{success_action_redirect}?bucket={bucket_name}&key={quote(object_key)}&etag={meta.etag}"
|
||||
return Response(status=303, headers={"Location": redirect_url})
|
||||
@@ -2740,9 +2776,14 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
except StorageError as exc:
|
||||
return _error_response("InternalError", str(exc), 500)
|
||||
else:
|
||||
stat = path.stat()
|
||||
file_size = stat.st_size
|
||||
etag = storage._compute_etag(path)
|
||||
try:
|
||||
stat = path.stat()
|
||||
file_size = stat.st_size
|
||||
etag = storage._compute_etag(path)
|
||||
except PermissionError:
|
||||
return _error_response("AccessDenied", "Permission denied accessing object", 403)
|
||||
except OSError as exc:
|
||||
return _error_response("InternalError", f"Failed to access object: {exc}", 500)
|
||||
|
||||
if range_header:
|
||||
try:
|
||||
@@ -2783,13 +2824,22 @@ def object_handler(bucket_name: str, object_key: str):
|
||||
except StorageError as exc:
|
||||
return _error_response("InternalError", str(exc), 500)
|
||||
else:
|
||||
stat = path.stat()
|
||||
response = Response(status=200)
|
||||
etag = storage._compute_etag(path)
|
||||
try:
|
||||
stat = path.stat()
|
||||
response = Response(status=200)
|
||||
etag = storage._compute_etag(path)
|
||||
except PermissionError:
|
||||
return _error_response("AccessDenied", "Permission denied accessing object", 403)
|
||||
except OSError as exc:
|
||||
return _error_response("InternalError", f"Failed to access object: {exc}", 500)
|
||||
response.headers["Content-Type"] = mimetype
|
||||
logged_bytes = 0
|
||||
|
||||
_apply_object_headers(response, file_stat=path.stat() if not is_encrypted else None, metadata=metadata, etag=etag)
|
||||
try:
|
||||
file_stat = path.stat() if not is_encrypted else None
|
||||
except (PermissionError, OSError):
|
||||
file_stat = None
|
||||
_apply_object_headers(response, file_stat=file_stat, metadata=metadata, etag=etag)
|
||||
|
||||
if request.method == "GET":
|
||||
response_overrides = {
|
||||
|
||||
@@ -18,6 +18,18 @@ class EphemeralSecretStore:
|
||||
self._store[token] = (payload, expires_at)
|
||||
return token
|
||||
|
||||
def peek(self, token: str | None) -> Any | None:
|
||||
if not token:
|
||||
return None
|
||||
entry = self._store.get(token)
|
||||
if not entry:
|
||||
return None
|
||||
payload, expires_at = entry
|
||||
if expires_at < time.time():
|
||||
self._store.pop(token, None)
|
||||
return None
|
||||
return payload
|
||||
|
||||
def pop(self, token: str | None) -> Any | None:
|
||||
if not token:
|
||||
return None
|
||||
|
||||
218
app/storage.py
218
app/storage.py
@@ -177,7 +177,7 @@ class ObjectStorage:
|
||||
self.root = Path(root)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
self._ensure_system_roots()
|
||||
self._object_cache: OrderedDict[str, tuple[Dict[str, ObjectMeta], float]] = OrderedDict()
|
||||
self._object_cache: OrderedDict[str, tuple[Dict[str, ObjectMeta], float, float]] = OrderedDict()
|
||||
self._cache_lock = threading.Lock()
|
||||
self._bucket_locks: Dict[str, threading.Lock] = {}
|
||||
self._cache_version: Dict[str, int] = {}
|
||||
@@ -186,6 +186,7 @@ class ObjectStorage:
|
||||
self._cache_ttl = cache_ttl
|
||||
self._object_cache_max_size = object_cache_max_size
|
||||
self._object_key_max_length_bytes = object_key_max_length_bytes
|
||||
self._sorted_key_cache: Dict[str, tuple[list[str], int]] = {}
|
||||
|
||||
def _get_bucket_lock(self, bucket_id: str) -> threading.Lock:
|
||||
"""Get or create a lock for a specific bucket. Reduces global lock contention."""
|
||||
@@ -243,10 +244,15 @@ class ObjectStorage:
|
||||
raise BucketNotFoundError("Bucket does not exist")
|
||||
|
||||
cache_path = self._system_bucket_root(bucket_name) / "stats.json"
|
||||
cached_stats = None
|
||||
cache_fresh = False
|
||||
|
||||
if cache_path.exists():
|
||||
try:
|
||||
if time.time() - cache_path.stat().st_mtime < cache_ttl:
|
||||
return json.loads(cache_path.read_text(encoding="utf-8"))
|
||||
cache_fresh = time.time() - cache_path.stat().st_mtime < cache_ttl
|
||||
cached_stats = json.loads(cache_path.read_text(encoding="utf-8"))
|
||||
if cache_fresh:
|
||||
return cached_stats
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
|
||||
@@ -255,24 +261,33 @@ class ObjectStorage:
|
||||
version_count = 0
|
||||
version_bytes = 0
|
||||
|
||||
for path in bucket_path.rglob("*"):
|
||||
if path.is_file():
|
||||
rel = path.relative_to(bucket_path)
|
||||
if not rel.parts:
|
||||
continue
|
||||
top_folder = rel.parts[0]
|
||||
if top_folder not in self.INTERNAL_FOLDERS:
|
||||
stat = path.stat()
|
||||
object_count += 1
|
||||
total_bytes += stat.st_size
|
||||
|
||||
versions_root = self._bucket_versions_root(bucket_name)
|
||||
if versions_root.exists():
|
||||
for path in versions_root.rglob("*.bin"):
|
||||
try:
|
||||
for path in bucket_path.rglob("*"):
|
||||
if path.is_file():
|
||||
stat = path.stat()
|
||||
version_count += 1
|
||||
version_bytes += stat.st_size
|
||||
rel = path.relative_to(bucket_path)
|
||||
if not rel.parts:
|
||||
continue
|
||||
top_folder = rel.parts[0]
|
||||
if top_folder not in self.INTERNAL_FOLDERS:
|
||||
stat = path.stat()
|
||||
object_count += 1
|
||||
total_bytes += stat.st_size
|
||||
|
||||
versions_root = self._bucket_versions_root(bucket_name)
|
||||
if versions_root.exists():
|
||||
for path in versions_root.rglob("*.bin"):
|
||||
if path.is_file():
|
||||
stat = path.stat()
|
||||
version_count += 1
|
||||
version_bytes += stat.st_size
|
||||
except OSError:
|
||||
if cached_stats is not None:
|
||||
return cached_stats
|
||||
raise
|
||||
|
||||
existing_serial = 0
|
||||
if cached_stats is not None:
|
||||
existing_serial = cached_stats.get("_cache_serial", 0)
|
||||
|
||||
stats = {
|
||||
"objects": object_count,
|
||||
@@ -281,6 +296,7 @@ class ObjectStorage:
|
||||
"version_bytes": version_bytes,
|
||||
"total_objects": object_count + version_count,
|
||||
"total_bytes": total_bytes + version_bytes,
|
||||
"_cache_serial": existing_serial,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -299,6 +315,39 @@ class ObjectStorage:
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _update_bucket_stats_cache(
|
||||
self,
|
||||
bucket_id: str,
|
||||
*,
|
||||
bytes_delta: int = 0,
|
||||
objects_delta: int = 0,
|
||||
version_bytes_delta: int = 0,
|
||||
version_count_delta: int = 0,
|
||||
) -> None:
|
||||
"""Incrementally update cached bucket statistics instead of invalidating.
|
||||
|
||||
This avoids expensive full directory scans on every PUT/DELETE by
|
||||
adjusting the cached values directly. Also signals cross-process cache
|
||||
invalidation by incrementing _cache_serial.
|
||||
"""
|
||||
cache_path = self._system_bucket_root(bucket_id) / "stats.json"
|
||||
try:
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if cache_path.exists():
|
||||
data = json.loads(cache_path.read_text(encoding="utf-8"))
|
||||
else:
|
||||
data = {"objects": 0, "bytes": 0, "version_count": 0, "version_bytes": 0, "total_objects": 0, "total_bytes": 0, "_cache_serial": 0}
|
||||
data["objects"] = max(0, data.get("objects", 0) + objects_delta)
|
||||
data["bytes"] = max(0, data.get("bytes", 0) + bytes_delta)
|
||||
data["version_count"] = max(0, data.get("version_count", 0) + version_count_delta)
|
||||
data["version_bytes"] = max(0, data.get("version_bytes", 0) + version_bytes_delta)
|
||||
data["total_objects"] = max(0, data.get("total_objects", 0) + objects_delta + version_count_delta)
|
||||
data["total_bytes"] = max(0, data.get("total_bytes", 0) + bytes_delta + version_bytes_delta)
|
||||
data["_cache_serial"] = data.get("_cache_serial", 0) + 1
|
||||
cache_path.write_text(json.dumps(data), encoding="utf-8")
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
|
||||
def delete_bucket(self, bucket_name: str) -> None:
|
||||
bucket_path = self._bucket_path(bucket_name)
|
||||
if not bucket_path.exists():
|
||||
@@ -333,6 +382,8 @@ class ObjectStorage:
|
||||
Returns:
|
||||
ListObjectsResult with objects, truncation status, and continuation token
|
||||
"""
|
||||
import bisect
|
||||
|
||||
bucket_path = self._bucket_path(bucket_name)
|
||||
if not bucket_path.exists():
|
||||
raise BucketNotFoundError("Bucket does not exist")
|
||||
@@ -340,15 +391,26 @@ class ObjectStorage:
|
||||
|
||||
object_cache = self._get_object_cache(bucket_id, bucket_path)
|
||||
|
||||
all_keys = sorted(object_cache.keys())
|
||||
cache_version = self._cache_version.get(bucket_id, 0)
|
||||
cached_entry = self._sorted_key_cache.get(bucket_id)
|
||||
if cached_entry and cached_entry[1] == cache_version:
|
||||
all_keys = cached_entry[0]
|
||||
else:
|
||||
all_keys = sorted(object_cache.keys())
|
||||
self._sorted_key_cache[bucket_id] = (all_keys, cache_version)
|
||||
|
||||
if prefix:
|
||||
all_keys = [k for k in all_keys if k.startswith(prefix)]
|
||||
lo = bisect.bisect_left(all_keys, prefix)
|
||||
hi = len(all_keys)
|
||||
for i in range(lo, len(all_keys)):
|
||||
if not all_keys[i].startswith(prefix):
|
||||
hi = i
|
||||
break
|
||||
all_keys = all_keys[lo:hi]
|
||||
|
||||
total_count = len(all_keys)
|
||||
start_index = 0
|
||||
if continuation_token:
|
||||
import bisect
|
||||
start_index = bisect.bisect_right(all_keys, continuation_token)
|
||||
if start_index >= total_count:
|
||||
return ListObjectsResult(
|
||||
@@ -403,7 +465,9 @@ class ObjectStorage:
|
||||
is_overwrite = destination.exists()
|
||||
existing_size = destination.stat().st_size if is_overwrite else 0
|
||||
|
||||
archived_version_size = 0
|
||||
if self._is_versioning_enabled(bucket_path) and is_overwrite:
|
||||
archived_version_size = existing_size
|
||||
self._archive_current_version(bucket_id, safe_key, reason="overwrite")
|
||||
|
||||
tmp_dir = self._system_root_path() / self.SYSTEM_TMP_DIR
|
||||
@@ -416,11 +480,10 @@ class ObjectStorage:
|
||||
shutil.copyfileobj(_HashingReader(stream, checksum), target)
|
||||
|
||||
new_size = tmp_path.stat().st_size
|
||||
size_delta = new_size - existing_size
|
||||
object_delta = 0 if is_overwrite else 1
|
||||
|
||||
if enforce_quota:
|
||||
size_delta = new_size - existing_size
|
||||
object_delta = 0 if is_overwrite else 1
|
||||
|
||||
quota_check = self.check_quota(
|
||||
bucket_name,
|
||||
additional_bytes=max(0, size_delta),
|
||||
@@ -448,7 +511,13 @@ class ObjectStorage:
|
||||
combined_meta = {**internal_meta, **(metadata or {})}
|
||||
self._write_metadata(bucket_id, safe_key, combined_meta)
|
||||
|
||||
self._invalidate_bucket_stats_cache(bucket_id)
|
||||
self._update_bucket_stats_cache(
|
||||
bucket_id,
|
||||
bytes_delta=size_delta,
|
||||
objects_delta=object_delta,
|
||||
version_bytes_delta=archived_version_size,
|
||||
version_count_delta=1 if archived_version_size > 0 else 0,
|
||||
)
|
||||
|
||||
obj_meta = ObjectMeta(
|
||||
key=safe_key.as_posix(),
|
||||
@@ -463,7 +532,7 @@ class ObjectStorage:
|
||||
|
||||
def get_object_path(self, bucket_name: str, object_key: str) -> Path:
|
||||
path = self._object_path(bucket_name, object_key)
|
||||
if not path.exists():
|
||||
if not path.is_file():
|
||||
raise ObjectNotFoundError("Object not found")
|
||||
return path
|
||||
|
||||
@@ -498,15 +567,24 @@ class ObjectStorage:
|
||||
path = self._object_path(bucket_name, object_key)
|
||||
if not path.exists():
|
||||
return
|
||||
deleted_size = path.stat().st_size
|
||||
safe_key = path.relative_to(bucket_path)
|
||||
bucket_id = bucket_path.name
|
||||
archived_version_size = 0
|
||||
if self._is_versioning_enabled(bucket_path):
|
||||
archived_version_size = deleted_size
|
||||
self._archive_current_version(bucket_id, safe_key, reason="delete")
|
||||
rel = path.relative_to(bucket_path)
|
||||
self._safe_unlink(path)
|
||||
self._delete_metadata(bucket_id, rel)
|
||||
|
||||
self._invalidate_bucket_stats_cache(bucket_id)
|
||||
self._update_bucket_stats_cache(
|
||||
bucket_id,
|
||||
bytes_delta=-deleted_size,
|
||||
objects_delta=-1,
|
||||
version_bytes_delta=archived_version_size,
|
||||
version_count_delta=1 if archived_version_size > 0 else 0,
|
||||
)
|
||||
self._update_object_cache_entry(bucket_id, safe_key.as_posix(), None)
|
||||
self._cleanup_empty_parents(path, bucket_path)
|
||||
|
||||
@@ -828,7 +906,12 @@ class ObjectStorage:
|
||||
if not isinstance(metadata, dict):
|
||||
metadata = {}
|
||||
destination = bucket_path / safe_key
|
||||
if self._is_versioning_enabled(bucket_path) and destination.exists():
|
||||
restored_size = data_path.stat().st_size
|
||||
is_overwrite = destination.exists()
|
||||
existing_size = destination.stat().st_size if is_overwrite else 0
|
||||
archived_version_size = 0
|
||||
if self._is_versioning_enabled(bucket_path) and is_overwrite:
|
||||
archived_version_size = existing_size
|
||||
self._archive_current_version(bucket_id, safe_key, reason="restore-overwrite")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(data_path, destination)
|
||||
@@ -837,7 +920,13 @@ class ObjectStorage:
|
||||
else:
|
||||
self._delete_metadata(bucket_id, safe_key)
|
||||
stat = destination.stat()
|
||||
self._invalidate_bucket_stats_cache(bucket_id)
|
||||
self._update_bucket_stats_cache(
|
||||
bucket_id,
|
||||
bytes_delta=restored_size - existing_size,
|
||||
objects_delta=0 if is_overwrite else 1,
|
||||
version_bytes_delta=archived_version_size,
|
||||
version_count_delta=1 if archived_version_size > 0 else 0,
|
||||
)
|
||||
return ObjectMeta(
|
||||
key=safe_key.as_posix(),
|
||||
size=stat.st_size,
|
||||
@@ -861,6 +950,7 @@ class ObjectStorage:
|
||||
meta_path = legacy_version_dir / f"{version_id}.json"
|
||||
if not data_path.exists() and not meta_path.exists():
|
||||
raise StorageError(f"Version {version_id} not found")
|
||||
deleted_version_size = data_path.stat().st_size if data_path.exists() else 0
|
||||
if data_path.exists():
|
||||
data_path.unlink()
|
||||
if meta_path.exists():
|
||||
@@ -868,6 +958,12 @@ class ObjectStorage:
|
||||
parent = data_path.parent
|
||||
if parent.exists() and not any(parent.iterdir()):
|
||||
parent.rmdir()
|
||||
if deleted_version_size > 0:
|
||||
self._update_bucket_stats_cache(
|
||||
bucket_id,
|
||||
version_bytes_delta=-deleted_version_size,
|
||||
version_count_delta=-1,
|
||||
)
|
||||
|
||||
def list_orphaned_objects(self, bucket_name: str) -> List[Dict[str, Any]]:
|
||||
bucket_path = self._bucket_path(bucket_name)
|
||||
@@ -1167,11 +1263,11 @@ class ObjectStorage:
|
||||
|
||||
is_overwrite = destination.exists()
|
||||
existing_size = destination.stat().st_size if is_overwrite else 0
|
||||
size_delta = total_size - existing_size
|
||||
object_delta = 0 if is_overwrite else 1
|
||||
versioning_enabled = self._is_versioning_enabled(bucket_path)
|
||||
|
||||
if enforce_quota:
|
||||
size_delta = total_size - existing_size
|
||||
object_delta = 0 if is_overwrite else 1
|
||||
|
||||
quota_check = self.check_quota(
|
||||
bucket_name,
|
||||
additional_bytes=max(0, size_delta),
|
||||
@@ -1188,9 +1284,11 @@ class ObjectStorage:
|
||||
|
||||
lock_file_path = self._system_bucket_root(bucket_id) / "locks" / f"{safe_key.as_posix().replace('/', '_')}.lock"
|
||||
|
||||
archived_version_size = 0
|
||||
try:
|
||||
with _atomic_lock_file(lock_file_path):
|
||||
if self._is_versioning_enabled(bucket_path) and destination.exists():
|
||||
if versioning_enabled and destination.exists():
|
||||
archived_version_size = destination.stat().st_size
|
||||
self._archive_current_version(bucket_id, safe_key, reason="overwrite")
|
||||
checksum = hashlib.md5()
|
||||
with destination.open("wb") as target:
|
||||
@@ -1210,7 +1308,13 @@ class ObjectStorage:
|
||||
|
||||
shutil.rmtree(upload_root, ignore_errors=True)
|
||||
|
||||
self._invalidate_bucket_stats_cache(bucket_id)
|
||||
self._update_bucket_stats_cache(
|
||||
bucket_id,
|
||||
bytes_delta=size_delta,
|
||||
objects_delta=object_delta,
|
||||
version_bytes_delta=archived_version_size,
|
||||
version_count_delta=1 if archived_version_size > 0 else 0,
|
||||
)
|
||||
|
||||
stat = destination.stat()
|
||||
etag = checksum.hexdigest()
|
||||
@@ -1469,7 +1573,8 @@ class ObjectStorage:
|
||||
|
||||
if meta_files:
|
||||
meta_cache = {}
|
||||
with ThreadPoolExecutor(max_workers=min(64, len(meta_files))) as executor:
|
||||
max_workers = min((os.cpu_count() or 4) * 2, len(meta_files), 16)
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
for key, etag in executor.map(read_meta_file, meta_files):
|
||||
if etag:
|
||||
meta_cache[key] = etag
|
||||
@@ -1522,38 +1627,46 @@ class ObjectStorage:
|
||||
|
||||
Uses LRU eviction to prevent unbounded cache growth.
|
||||
Thread-safe with per-bucket locks to reduce contention.
|
||||
Checks stats.json for cross-process cache invalidation.
|
||||
"""
|
||||
now = time.time()
|
||||
current_stats_mtime = self._get_cache_marker_mtime(bucket_id)
|
||||
|
||||
with self._cache_lock:
|
||||
cached = self._object_cache.get(bucket_id)
|
||||
if cached:
|
||||
objects, timestamp = cached
|
||||
if now - timestamp < self._cache_ttl:
|
||||
objects, timestamp, cached_stats_mtime = cached
|
||||
if now - timestamp < self._cache_ttl and current_stats_mtime == cached_stats_mtime:
|
||||
self._object_cache.move_to_end(bucket_id)
|
||||
return objects
|
||||
cache_version = self._cache_version.get(bucket_id, 0)
|
||||
|
||||
bucket_lock = self._get_bucket_lock(bucket_id)
|
||||
with bucket_lock:
|
||||
current_stats_mtime = self._get_cache_marker_mtime(bucket_id)
|
||||
with self._cache_lock:
|
||||
cached = self._object_cache.get(bucket_id)
|
||||
if cached:
|
||||
objects, timestamp = cached
|
||||
if now - timestamp < self._cache_ttl:
|
||||
objects, timestamp, cached_stats_mtime = cached
|
||||
if now - timestamp < self._cache_ttl and current_stats_mtime == cached_stats_mtime:
|
||||
self._object_cache.move_to_end(bucket_id)
|
||||
return objects
|
||||
|
||||
objects = self._build_object_cache(bucket_path)
|
||||
new_stats_mtime = self._get_cache_marker_mtime(bucket_id)
|
||||
|
||||
with self._cache_lock:
|
||||
current_version = self._cache_version.get(bucket_id, 0)
|
||||
if current_version != cache_version:
|
||||
objects = self._build_object_cache(bucket_path)
|
||||
new_stats_mtime = self._get_cache_marker_mtime(bucket_id)
|
||||
while len(self._object_cache) >= self._object_cache_max_size:
|
||||
self._object_cache.popitem(last=False)
|
||||
|
||||
self._object_cache[bucket_id] = (objects, time.time())
|
||||
self._object_cache[bucket_id] = (objects, time.time(), new_stats_mtime)
|
||||
self._object_cache.move_to_end(bucket_id)
|
||||
self._cache_version[bucket_id] = current_version + 1
|
||||
self._sorted_key_cache.pop(bucket_id, None)
|
||||
|
||||
return objects
|
||||
|
||||
@@ -1561,6 +1674,7 @@ class ObjectStorage:
|
||||
"""Invalidate the object cache and etag index for a bucket.
|
||||
|
||||
Increments version counter to signal stale reads.
|
||||
Cross-process invalidation is handled by checking stats.json mtime.
|
||||
"""
|
||||
with self._cache_lock:
|
||||
self._object_cache.pop(bucket_id, None)
|
||||
@@ -1572,19 +1686,37 @@ class ObjectStorage:
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _get_cache_marker_mtime(self, bucket_id: str) -> float:
|
||||
"""Get a cache marker combining serial and object count for cross-process invalidation.
|
||||
|
||||
Returns a combined value that changes if either _cache_serial or object count changes.
|
||||
This handles cases where the serial was reset but object count differs.
|
||||
"""
|
||||
stats_path = self._system_bucket_root(bucket_id) / "stats.json"
|
||||
try:
|
||||
data = json.loads(stats_path.read_text(encoding="utf-8"))
|
||||
serial = data.get("_cache_serial", 0)
|
||||
count = data.get("objects", 0)
|
||||
return float(serial * 1000000 + count)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return 0
|
||||
|
||||
def _update_object_cache_entry(self, bucket_id: str, key: str, meta: Optional[ObjectMeta]) -> None:
|
||||
"""Update a single entry in the object cache instead of invalidating the whole cache.
|
||||
|
||||
This is a performance optimization - lazy update instead of full invalidation.
|
||||
Cross-process invalidation is handled by checking stats.json mtime.
|
||||
"""
|
||||
with self._cache_lock:
|
||||
cached = self._object_cache.get(bucket_id)
|
||||
if cached:
|
||||
objects, timestamp = cached
|
||||
objects, timestamp, stats_mtime = cached
|
||||
if meta is None:
|
||||
objects.pop(key, None)
|
||||
else:
|
||||
objects[key] = meta
|
||||
self._cache_version[bucket_id] = self._cache_version.get(bucket_id, 0) + 1
|
||||
self._sorted_key_cache.pop(bucket_id, None)
|
||||
|
||||
def warm_cache(self, bucket_names: Optional[List[str]] = None) -> None:
|
||||
"""Pre-warm the object cache for specified buckets or all buckets.
|
||||
|
||||
29
app/ui.py
29
app/ui.py
@@ -220,13 +220,16 @@ def _bucket_access_descriptor(policy: dict[str, Any] | None) -> tuple[str, str]:
|
||||
|
||||
|
||||
def _current_principal():
|
||||
creds = session.get("credentials")
|
||||
token = session.get("cred_token")
|
||||
creds = _secret_store().peek(token) if token else None
|
||||
if not creds:
|
||||
return None
|
||||
try:
|
||||
return _iam().authenticate(creds["access_key"], creds["secret_key"])
|
||||
except IamError:
|
||||
session.pop("credentials", None)
|
||||
session.pop("cred_token", None)
|
||||
if token:
|
||||
_secret_store().pop(token)
|
||||
return None
|
||||
|
||||
|
||||
@@ -251,7 +254,8 @@ def _authorize_ui(principal, bucket_name: str | None, action: str, *, object_key
|
||||
|
||||
|
||||
def _api_headers() -> dict[str, str]:
|
||||
creds = session.get("credentials") or {}
|
||||
token = session.get("cred_token")
|
||||
creds = _secret_store().peek(token) or {}
|
||||
return {
|
||||
"X-Access-Key": creds.get("access_key", ""),
|
||||
"X-Secret-Key": creds.get("secret_key", ""),
|
||||
@@ -296,7 +300,10 @@ def login():
|
||||
except IamError as exc:
|
||||
flash(_friendly_error_message(exc), "danger")
|
||||
return render_template("login.html")
|
||||
session["credentials"] = {"access_key": access_key, "secret_key": secret_key}
|
||||
creds = {"access_key": access_key, "secret_key": secret_key}
|
||||
ttl = int(current_app.permanent_session_lifetime.total_seconds())
|
||||
token = _secret_store().remember(creds, ttl=ttl)
|
||||
session["cred_token"] = token
|
||||
session.permanent = True
|
||||
flash(f"Welcome back, {principal.display_name}", "success")
|
||||
return redirect(url_for("ui.buckets_overview"))
|
||||
@@ -305,7 +312,9 @@ def login():
|
||||
|
||||
@ui_bp.post("/logout")
|
||||
def logout():
|
||||
session.pop("credentials", None)
|
||||
token = session.pop("cred_token", None)
|
||||
if token:
|
||||
_secret_store().pop(token)
|
||||
flash("Signed out", "info")
|
||||
return redirect(url_for("ui.login"))
|
||||
|
||||
@@ -542,7 +551,10 @@ def list_bucket_objects(bucket_name: str):
|
||||
except IamError as exc:
|
||||
return jsonify({"error": str(exc)}), 403
|
||||
|
||||
max_keys = min(int(request.args.get("max_keys", 1000)), 100000)
|
||||
try:
|
||||
max_keys = min(int(request.args.get("max_keys", 1000)), 100000)
|
||||
except ValueError:
|
||||
return jsonify({"error": "max_keys must be an integer"}), 400
|
||||
continuation_token = request.args.get("continuation_token") or None
|
||||
prefix = request.args.get("prefix") or None
|
||||
|
||||
@@ -582,7 +594,7 @@ def list_bucket_objects(bucket_name: str):
|
||||
"etag": obj.etag,
|
||||
})
|
||||
|
||||
return jsonify({
|
||||
response = jsonify({
|
||||
"objects": objects_data,
|
||||
"is_truncated": result.is_truncated,
|
||||
"next_continuation_token": result.next_continuation_token,
|
||||
@@ -601,6 +613,8 @@ def list_bucket_objects(bucket_name: str):
|
||||
"metadata": metadata_template,
|
||||
},
|
||||
})
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
|
||||
|
||||
@ui_bp.get("/buckets/<bucket_name>/objects/stream")
|
||||
@@ -697,6 +711,7 @@ def stream_bucket_objects(bucket_name: str):
|
||||
headers={
|
||||
'Cache-Control': 'no-cache',
|
||||
'X-Accel-Buffering': 'no',
|
||||
'X-Stream-Response': 'true',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
APP_VERSION = "0.2.4"
|
||||
APP_VERSION = "0.2.6"
|
||||
|
||||
|
||||
def get_version() -> str:
|
||||
|
||||
6
docs.md
6
docs.md
@@ -619,13 +619,15 @@ MyFSIO implements a comprehensive Identity and Access Management (IAM) system th
|
||||
|
||||
### Getting Started
|
||||
|
||||
1. On first boot, `data/.myfsio.sys/config/iam.json` is seeded with `localadmin / localadmin` that has wildcard access.
|
||||
2. Sign into the UI using those credentials, then open **IAM**:
|
||||
1. On first boot, `data/.myfsio.sys/config/iam.json` is created with a randomly generated admin user. The access key and secret key are printed to the console during first startup. If you miss it, check the `iam.json` file directly—credentials are stored in plaintext.
|
||||
2. Sign into the UI using the generated credentials, then open **IAM**:
|
||||
- **Create user**: supply a display name and optional JSON inline policy array.
|
||||
- **Rotate secret**: generates a new secret key; the UI surfaces it once.
|
||||
- **Policy editor**: select a user, paste an array of objects (`{"bucket": "*", "actions": ["list", "read"]}`), and submit. Alias support includes AWS-style verbs (e.g., `s3:GetObject`).
|
||||
3. Wildcard action `iam:*` is supported for admin user definitions.
|
||||
|
||||
> **Breaking Change (v0.2.0+):** Previous versions used fixed default credentials (`localadmin/localadmin`). If upgrading from an older version, your existing credentials remain unchanged, but new installations will generate random credentials.
|
||||
|
||||
### Authentication
|
||||
|
||||
The API expects every request to include authentication headers. The UI persists them in the Flask session after login.
|
||||
|
||||
5
run.py
5
run.py
@@ -5,6 +5,7 @@ import argparse
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
import multiprocessing
|
||||
from multiprocessing import Process
|
||||
from pathlib import Path
|
||||
|
||||
@@ -87,6 +88,10 @@ def serve_ui(port: int, prod: bool = False, config: Optional[AppConfig] = None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
multiprocessing.freeze_support()
|
||||
if _is_frozen():
|
||||
multiprocessing.set_start_method("spawn", force=True)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Run the S3 clone services.")
|
||||
parser.add_argument("--mode", choices=["api", "ui", "both"], default="both")
|
||||
parser.add_argument("--api-port", type=int, default=5000)
|
||||
|
||||
@@ -192,31 +192,86 @@ cat > "$INSTALL_DIR/myfsio.env" << EOF
|
||||
# Generated by install.sh on $(date)
|
||||
# Documentation: https://go.jzwsite.com/myfsio
|
||||
|
||||
# Storage paths
|
||||
# =============================================================================
|
||||
# STORAGE PATHS
|
||||
# =============================================================================
|
||||
STORAGE_ROOT=$DATA_DIR
|
||||
LOG_DIR=$LOG_DIR
|
||||
|
||||
# Network
|
||||
# =============================================================================
|
||||
# NETWORK
|
||||
# =============================================================================
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=$API_PORT
|
||||
|
||||
# Security - CHANGE IN PRODUCTION
|
||||
SECRET_KEY=$SECRET_KEY
|
||||
CORS_ORIGINS=*
|
||||
|
||||
# Public URL (set this if behind a reverse proxy)
|
||||
# Public URL (set this if behind a reverse proxy for presigned URLs)
|
||||
$(if [[ -n "$API_URL" ]]; then echo "API_BASE_URL=$API_URL"; else echo "# API_BASE_URL=https://s3.example.com"; fi)
|
||||
|
||||
# Logging
|
||||
# =============================================================================
|
||||
# SECURITY
|
||||
# =============================================================================
|
||||
# Secret key for session signing (auto-generated if not set)
|
||||
SECRET_KEY=$SECRET_KEY
|
||||
|
||||
# CORS settings - restrict in production
|
||||
CORS_ORIGINS=*
|
||||
|
||||
# Brute-force protection
|
||||
AUTH_MAX_ATTEMPTS=5
|
||||
AUTH_LOCKOUT_MINUTES=15
|
||||
|
||||
# Reverse proxy settings (set to number of trusted proxies in front)
|
||||
# NUM_TRUSTED_PROXIES=1
|
||||
|
||||
# Allow internal admin endpoints (only enable on trusted networks)
|
||||
# ALLOW_INTERNAL_ENDPOINTS=false
|
||||
|
||||
# Allowed hosts for redirects (comma-separated, empty = restrict all)
|
||||
# ALLOWED_REDIRECT_HOSTS=
|
||||
|
||||
# =============================================================================
|
||||
# LOGGING
|
||||
# =============================================================================
|
||||
LOG_LEVEL=INFO
|
||||
LOG_TO_FILE=true
|
||||
|
||||
# Rate limiting
|
||||
# =============================================================================
|
||||
# RATE LIMITING
|
||||
# =============================================================================
|
||||
RATE_LIMIT_DEFAULT=200 per minute
|
||||
# RATE_LIMIT_LIST_BUCKETS=60 per minute
|
||||
# RATE_LIMIT_BUCKET_OPS=120 per minute
|
||||
# RATE_LIMIT_OBJECT_OPS=240 per minute
|
||||
# RATE_LIMIT_ADMIN=60 per minute
|
||||
|
||||
# Optional: Encryption (uncomment to enable)
|
||||
# =============================================================================
|
||||
# SERVER TUNING (0 = auto-detect based on system resources)
|
||||
# =============================================================================
|
||||
# SERVER_THREADS=0
|
||||
# SERVER_CONNECTION_LIMIT=0
|
||||
# SERVER_BACKLOG=0
|
||||
# SERVER_CHANNEL_TIMEOUT=120
|
||||
|
||||
# =============================================================================
|
||||
# ENCRYPTION (uncomment to enable)
|
||||
# =============================================================================
|
||||
# ENCRYPTION_ENABLED=true
|
||||
# KMS_ENABLED=true
|
||||
|
||||
# =============================================================================
|
||||
# SITE SYNC / REPLICATION (for multi-site deployments)
|
||||
# =============================================================================
|
||||
# SITE_ID=site-1
|
||||
# SITE_ENDPOINT=https://s3-site1.example.com
|
||||
# SITE_REGION=us-east-1
|
||||
# SITE_SYNC_ENABLED=false
|
||||
|
||||
# =============================================================================
|
||||
# OPTIONAL FEATURES
|
||||
# =============================================================================
|
||||
# LIFECYCLE_ENABLED=false
|
||||
# METRICS_HISTORY_ENABLED=false
|
||||
# OPERATION_METRICS_ENABLED=false
|
||||
EOF
|
||||
chmod 600 "$INSTALL_DIR/myfsio.env"
|
||||
echo " [OK] Created $INSTALL_DIR/myfsio.env"
|
||||
@@ -317,11 +372,36 @@ if [[ "$SKIP_SYSTEMD" != true ]]; then
|
||||
fi
|
||||
echo ""
|
||||
|
||||
sleep 2
|
||||
echo " Waiting for service initialization..."
|
||||
sleep 3
|
||||
|
||||
echo " Service Status:"
|
||||
echo " ---------------"
|
||||
if systemctl is-active --quiet myfsio; then
|
||||
echo " [OK] MyFSIO is running"
|
||||
|
||||
IAM_FILE="$DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
if [[ -f "$IAM_FILE" ]]; then
|
||||
echo ""
|
||||
echo " ============================================"
|
||||
echo " ADMIN CREDENTIALS (save these securely!)"
|
||||
echo " ============================================"
|
||||
if command -v jq &>/dev/null; then
|
||||
ACCESS_KEY=$(jq -r '.users[0].access_key' "$IAM_FILE" 2>/dev/null)
|
||||
SECRET_KEY=$(jq -r '.users[0].secret_key' "$IAM_FILE" 2>/dev/null)
|
||||
else
|
||||
ACCESS_KEY=$(grep -o '"access_key"[[:space:]]*:[[:space:]]*"[^"]*"' "$IAM_FILE" | head -1 | sed 's/.*"\([^"]*\)"$/\1/')
|
||||
SECRET_KEY=$(grep -o '"secret_key"[[:space:]]*:[[:space:]]*"[^"]*"' "$IAM_FILE" | head -1 | sed 's/.*"\([^"]*\)"$/\1/')
|
||||
fi
|
||||
if [[ -n "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then
|
||||
echo " Access Key: $ACCESS_KEY"
|
||||
echo " Secret Key: $SECRET_KEY"
|
||||
else
|
||||
echo " [!] Could not parse credentials from $IAM_FILE"
|
||||
echo " Check the file manually or view service logs."
|
||||
fi
|
||||
echo " ============================================"
|
||||
fi
|
||||
else
|
||||
echo " [WARNING] MyFSIO may not have started correctly"
|
||||
echo " Check logs with: journalctl -u myfsio -f"
|
||||
@@ -346,19 +426,26 @@ echo "Access Points:"
|
||||
echo " API: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost"):$API_PORT"
|
||||
echo " UI: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost"):$UI_PORT/ui"
|
||||
echo ""
|
||||
echo "Default Credentials:"
|
||||
echo " Username: localadmin"
|
||||
echo " Password: localadmin"
|
||||
echo " [!] WARNING: Change these immediately after first login!"
|
||||
echo "Credentials:"
|
||||
echo " Admin credentials were shown above (if service was started)."
|
||||
echo " You can also find them in: $DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
echo ""
|
||||
echo "Configuration Files:"
|
||||
echo " Environment: $INSTALL_DIR/myfsio.env"
|
||||
echo " IAM Users: $DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
echo " Bucket Policies: $DATA_DIR/.myfsio.sys/config/bucket_policies.json"
|
||||
echo " Secret Key: $DATA_DIR/.myfsio.sys/config/.secret (auto-generated)"
|
||||
echo ""
|
||||
echo "Security Notes:"
|
||||
echo " - Rate limiting is enabled by default (200 req/min)"
|
||||
echo " - Brute-force protection: 5 attempts, 15 min lockout"
|
||||
echo " - Set CORS_ORIGINS to specific domains in production"
|
||||
echo " - Set NUM_TRUSTED_PROXIES if behind a reverse proxy"
|
||||
echo ""
|
||||
echo "Useful Commands:"
|
||||
echo " Check status: sudo systemctl status myfsio"
|
||||
echo " View logs: sudo journalctl -u myfsio -f"
|
||||
echo " Validate config: $INSTALL_DIR/myfsio --check-config"
|
||||
echo " Restart: sudo systemctl restart myfsio"
|
||||
echo " Stop: sudo systemctl stop myfsio"
|
||||
echo ""
|
||||
|
||||
@@ -88,7 +88,8 @@ echo "The following items will be removed:"
|
||||
echo ""
|
||||
echo " Install directory: $INSTALL_DIR"
|
||||
if [[ "$KEEP_DATA" != true ]]; then
|
||||
echo " Data directory: $DATA_DIR (ALL YOUR DATA WILL BE DELETED!)"
|
||||
echo " Data directory: $DATA_DIR"
|
||||
echo " [!] ALL DATA, IAM USERS, AND ENCRYPTION KEYS WILL BE DELETED!"
|
||||
else
|
||||
echo " Data directory: $DATA_DIR (WILL BE KEPT)"
|
||||
fi
|
||||
@@ -227,8 +228,15 @@ echo ""
|
||||
if [[ "$KEEP_DATA" == true ]]; then
|
||||
echo "Your data has been preserved at: $DATA_DIR"
|
||||
echo ""
|
||||
echo "To reinstall MyFSIO with existing data, run:"
|
||||
echo " curl -fsSL https://go.jzwsite.com/myfsio-install | sudo bash"
|
||||
echo "Preserved files include:"
|
||||
echo " - All buckets and objects"
|
||||
echo " - IAM configuration: $DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
echo " - Bucket policies: $DATA_DIR/.myfsio.sys/config/bucket_policies.json"
|
||||
echo " - Secret key: $DATA_DIR/.myfsio.sys/config/.secret"
|
||||
echo " - Encryption keys: $DATA_DIR/.myfsio.sys/keys/ (if encryption was enabled)"
|
||||
echo ""
|
||||
echo "To reinstall MyFSIO with existing data:"
|
||||
echo " ./install.sh --data-dir $DATA_DIR"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
|
||||
@@ -182,6 +182,9 @@
|
||||
let visibleItems = [];
|
||||
let renderedRange = { start: 0, end: 0 };
|
||||
|
||||
let memoizedVisibleItems = null;
|
||||
let memoizedInputs = { objectCount: -1, prefix: null, filterTerm: null };
|
||||
|
||||
const createObjectRow = (obj, displayKey = null) => {
|
||||
const tr = document.createElement('tr');
|
||||
tr.dataset.objectRow = '';
|
||||
@@ -340,7 +343,21 @@
|
||||
}
|
||||
};
|
||||
|
||||
const computeVisibleItems = () => {
|
||||
const computeVisibleItems = (forceRecompute = false) => {
|
||||
const currentInputs = {
|
||||
objectCount: allObjects.length,
|
||||
prefix: currentPrefix,
|
||||
filterTerm: currentFilterTerm
|
||||
};
|
||||
|
||||
if (!forceRecompute &&
|
||||
memoizedVisibleItems !== null &&
|
||||
memoizedInputs.objectCount === currentInputs.objectCount &&
|
||||
memoizedInputs.prefix === currentInputs.prefix &&
|
||||
memoizedInputs.filterTerm === currentInputs.filterTerm) {
|
||||
return memoizedVisibleItems;
|
||||
}
|
||||
|
||||
const items = [];
|
||||
const folders = new Set();
|
||||
|
||||
@@ -381,6 +398,8 @@
|
||||
return aKey.localeCompare(bKey);
|
||||
});
|
||||
|
||||
memoizedVisibleItems = items;
|
||||
memoizedInputs = currentInputs;
|
||||
return items;
|
||||
};
|
||||
|
||||
@@ -533,6 +552,8 @@
|
||||
loadedObjectCount = 0;
|
||||
totalObjectCount = 0;
|
||||
allObjects = [];
|
||||
memoizedVisibleItems = null;
|
||||
memoizedInputs = { objectCount: -1, prefix: null, filterTerm: null };
|
||||
pendingStreamObjects = [];
|
||||
|
||||
streamAbortController = new AbortController();
|
||||
@@ -643,6 +664,8 @@
|
||||
loadedObjectCount = 0;
|
||||
totalObjectCount = 0;
|
||||
allObjects = [];
|
||||
memoizedVisibleItems = null;
|
||||
memoizedInputs = { objectCount: -1, prefix: null, filterTerm: null };
|
||||
}
|
||||
|
||||
if (append && loadMoreSpinner) {
|
||||
@@ -985,13 +1008,15 @@
|
||||
};
|
||||
|
||||
const navigateToFolder = (prefix) => {
|
||||
if (streamAbortController) {
|
||||
streamAbortController.abort();
|
||||
streamAbortController = null;
|
||||
}
|
||||
|
||||
currentPrefix = prefix;
|
||||
|
||||
if (scrollContainer) scrollContainer.scrollTop = 0;
|
||||
|
||||
refreshVirtualList();
|
||||
renderBreadcrumb(prefix);
|
||||
|
||||
selectedRows.clear();
|
||||
|
||||
if (typeof updateBulkDeleteState === 'function') {
|
||||
@@ -1001,6 +1026,9 @@
|
||||
if (previewPanel) previewPanel.classList.add('d-none');
|
||||
if (previewEmpty) previewEmpty.classList.remove('d-none');
|
||||
activeRow = null;
|
||||
|
||||
isLoadingObjects = false;
|
||||
loadObjects(false);
|
||||
};
|
||||
|
||||
const renderObjectsView = () => {
|
||||
|
||||
@@ -451,10 +451,10 @@ sudo journalctl -u myfsio -f # View logs</code></pre>
|
||||
<span class="docs-section-kicker">03</span>
|
||||
<h2 class="h4 mb-0">Authenticate & manage IAM</h2>
|
||||
</div>
|
||||
<p class="text-muted">MyFSIO seeds <code>data/.myfsio.sys/config/iam.json</code> with <code>localadmin/localadmin</code>. Sign in once, rotate it, then grant least-privilege access to teammates and tools.</p>
|
||||
<p class="text-muted">On first startup, MyFSIO generates random admin credentials and prints them to the console. Missed it? Check <code>data/.myfsio.sys/config/iam.json</code> directly—credentials are stored in plaintext.</p>
|
||||
<div class="docs-highlight mb-3">
|
||||
<ol class="mb-0">
|
||||
<li>Visit <code>/ui/login</code>, enter the bootstrap credentials, and rotate them immediately from the IAM page.</li>
|
||||
<li>Check the console output (or <code>iam.json</code>) for the generated <code>Access Key</code> and <code>Secret Key</code>, then visit <code>/ui/login</code>.</li>
|
||||
<li>Create additional users with descriptive display names and AWS-style inline policies (for example <code>{"bucket": "*", "actions": ["list", "read"]}</code>).</li>
|
||||
<li>Rotate secrets when sharing with CI jobs—new secrets display once and persist to <code>data/.myfsio.sys/config/iam.json</code>.</li>
|
||||
<li>Bucket policies layer on top of IAM. Apply Private/Public presets or paste custom JSON; changes reload instantly.</li>
|
||||
@@ -2136,8 +2136,8 @@ curl -X PUT "{{ api_base }}/<bucket>?tagging" \
|
||||
<code class="d-block">{{ api_base }}</code>
|
||||
</div>
|
||||
<div>
|
||||
<div class="small text-uppercase text-muted">Sample user</div>
|
||||
<code class="d-block">localadmin / localadmin</code>
|
||||
<div class="small text-uppercase text-muted">Initial credentials</div>
|
||||
<span class="text-muted small">Generated on first run (check console)</span>
|
||||
</div>
|
||||
<div>
|
||||
<div class="small text-uppercase text-muted">Logs</div>
|
||||
|
||||
@@ -398,6 +398,14 @@
|
||||
<option value="24" selected>Last 24 hours</option>
|
||||
<option value="168">Last 7 days</option>
|
||||
</select>
|
||||
<select class="form-select form-select-sm" id="maxDataPoints" style="width: auto;" title="Maximum data points to display">
|
||||
<option value="100">100 points</option>
|
||||
<option value="250">250 points</option>
|
||||
<option value="500" selected>500 points</option>
|
||||
<option value="1000">1000 points</option>
|
||||
<option value="2000">2000 points</option>
|
||||
<option value="0">Unlimited</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body p-4">
|
||||
@@ -817,8 +825,8 @@
|
||||
var diskChart = null;
|
||||
var historyStatus = document.getElementById('historyStatus');
|
||||
var timeRangeSelect = document.getElementById('historyTimeRange');
|
||||
var maxDataPointsSelect = document.getElementById('maxDataPoints');
|
||||
var historyTimer = null;
|
||||
var MAX_DATA_POINTS = 500;
|
||||
|
||||
function createChart(ctx, label, color) {
|
||||
return new Chart(ctx, {
|
||||
@@ -889,7 +897,8 @@
|
||||
if (historyStatus) historyStatus.textContent = 'No history data available yet. Data is recorded every ' + (data.interval_minutes || 5) + ' minutes.';
|
||||
return;
|
||||
}
|
||||
var history = data.history.slice(-MAX_DATA_POINTS);
|
||||
var maxPoints = maxDataPointsSelect ? parseInt(maxDataPointsSelect.value, 10) : 500;
|
||||
var history = maxPoints > 0 ? data.history.slice(-maxPoints) : data.history;
|
||||
var labels = history.map(function(h) { return formatTime(h.timestamp); });
|
||||
var cpuData = history.map(function(h) { return h.cpu_percent; });
|
||||
var memData = history.map(function(h) { return h.memory_percent; });
|
||||
@@ -927,6 +936,10 @@
|
||||
timeRangeSelect.addEventListener('change', loadHistory);
|
||||
}
|
||||
|
||||
if (maxDataPointsSelect) {
|
||||
maxDataPointsSelect.addEventListener('change', loadHistory);
|
||||
}
|
||||
|
||||
document.addEventListener('visibilitychange', function() {
|
||||
if (document.hidden) {
|
||||
if (historyTimer) clearInterval(historyTimer);
|
||||
|
||||
Reference in New Issue
Block a user