Compare commits
21 Commits
1c328ee3af
...
v0.3.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 0462a7b62e | |||
| 52660570c1 | |||
| 35f61313e0 | |||
| c470cfb576 | |||
| d96955deee | |||
| 85181f0be6 | |||
| d5ca7a8be1 | |||
| 476dc79e42 | |||
| bb6590fc5e | |||
| 899db3421b | |||
| caf01d6ada | |||
| bb366cb4cd | |||
| a2745ff2ee | |||
| 28cb656d94 | |||
| 3c44152fc6 | |||
| 397515edce | |||
| 980fced7e4 | |||
| bae5009ec4 | |||
| 233780617f | |||
| fd8fb21517 | |||
| c6cbe822e1 |
@@ -115,7 +115,7 @@ def create_app(
|
|||||||
|
|
||||||
storage = ObjectStorage(
|
storage = ObjectStorage(
|
||||||
Path(app.config["STORAGE_ROOT"]),
|
Path(app.config["STORAGE_ROOT"]),
|
||||||
cache_ttl=app.config.get("OBJECT_CACHE_TTL", 60),
|
cache_ttl=app.config.get("OBJECT_CACHE_TTL", 5),
|
||||||
object_cache_max_size=app.config.get("OBJECT_CACHE_MAX_SIZE", 100),
|
object_cache_max_size=app.config.get("OBJECT_CACHE_MAX_SIZE", 100),
|
||||||
bucket_config_cache_ttl=app.config.get("BUCKET_CONFIG_CACHE_TTL_SECONDS", 30.0),
|
bucket_config_cache_ttl=app.config.get("BUCKET_CONFIG_CACHE_TTL_SECONDS", 30.0),
|
||||||
object_key_max_length_bytes=app.config.get("OBJECT_KEY_MAX_LENGTH_BYTES", 1024),
|
object_key_max_length_bytes=app.config.get("OBJECT_KEY_MAX_LENGTH_BYTES", 1024),
|
||||||
|
|||||||
@@ -241,7 +241,7 @@ class AppConfig:
|
|||||||
cors_expose_headers = _csv(str(_get("CORS_EXPOSE_HEADERS", "*")), ["*"])
|
cors_expose_headers = _csv(str(_get("CORS_EXPOSE_HEADERS", "*")), ["*"])
|
||||||
session_lifetime_days = int(_get("SESSION_LIFETIME_DAYS", 30))
|
session_lifetime_days = int(_get("SESSION_LIFETIME_DAYS", 30))
|
||||||
bucket_stats_cache_ttl = int(_get("BUCKET_STATS_CACHE_TTL", 60))
|
bucket_stats_cache_ttl = int(_get("BUCKET_STATS_CACHE_TTL", 60))
|
||||||
object_cache_ttl = int(_get("OBJECT_CACHE_TTL", 60))
|
object_cache_ttl = int(_get("OBJECT_CACHE_TTL", 5))
|
||||||
|
|
||||||
encryption_enabled = str(_get("ENCRYPTION_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
encryption_enabled = str(_get("ENCRYPTION_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||||
encryption_keys_dir = storage_root / ".myfsio.sys" / "keys"
|
encryption_keys_dir = storage_root / ".myfsio.sys" / "keys"
|
||||||
|
|||||||
@@ -190,9 +190,6 @@ class EncryptedObjectStorage:
|
|||||||
def list_objects(self, bucket_name: str, **kwargs):
|
def list_objects(self, bucket_name: str, **kwargs):
|
||||||
return self.storage.list_objects(bucket_name, **kwargs)
|
return self.storage.list_objects(bucket_name, **kwargs)
|
||||||
|
|
||||||
def list_objects_shallow(self, bucket_name: str, **kwargs):
|
|
||||||
return self.storage.list_objects_shallow(bucket_name, **kwargs)
|
|
||||||
|
|
||||||
def list_objects_all(self, bucket_name: str):
|
def list_objects_all(self, bucket_name: str):
|
||||||
return self.storage.list_objects_all(bucket_name)
|
return self.storage.list_objects_all(bucket_name)
|
||||||
|
|
||||||
|
|||||||
@@ -2671,44 +2671,55 @@ def bucket_handler(bucket_name: str) -> Response:
|
|||||||
else:
|
else:
|
||||||
effective_start = marker
|
effective_start = marker
|
||||||
|
|
||||||
|
fetch_keys = max_keys * 10 if delimiter else max_keys
|
||||||
try:
|
try:
|
||||||
if delimiter:
|
list_result = storage.list_objects(
|
||||||
shallow_result = storage.list_objects_shallow(
|
bucket_name,
|
||||||
bucket_name,
|
max_keys=fetch_keys,
|
||||||
prefix=prefix,
|
continuation_token=effective_start or None,
|
||||||
delimiter=delimiter,
|
prefix=prefix or None,
|
||||||
max_keys=max_keys,
|
)
|
||||||
continuation_token=effective_start or None,
|
objects = list_result.objects
|
||||||
)
|
|
||||||
objects = shallow_result.objects
|
|
||||||
common_prefixes = shallow_result.common_prefixes
|
|
||||||
is_truncated = shallow_result.is_truncated
|
|
||||||
|
|
||||||
next_marker = shallow_result.next_continuation_token or ""
|
|
||||||
next_continuation_token = ""
|
|
||||||
if is_truncated and next_marker and list_type == "2":
|
|
||||||
next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
|
|
||||||
else:
|
|
||||||
list_result = storage.list_objects(
|
|
||||||
bucket_name,
|
|
||||||
max_keys=max_keys,
|
|
||||||
continuation_token=effective_start or None,
|
|
||||||
prefix=prefix or None,
|
|
||||||
)
|
|
||||||
objects = list_result.objects
|
|
||||||
common_prefixes = []
|
|
||||||
is_truncated = list_result.is_truncated
|
|
||||||
|
|
||||||
next_marker = ""
|
|
||||||
next_continuation_token = ""
|
|
||||||
if is_truncated:
|
|
||||||
if objects:
|
|
||||||
next_marker = objects[-1].key
|
|
||||||
if list_type == "2" and next_marker:
|
|
||||||
next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
|
|
||||||
except StorageError as exc:
|
except StorageError as exc:
|
||||||
return _error_response("NoSuchBucket", str(exc), 404)
|
return _error_response("NoSuchBucket", str(exc), 404)
|
||||||
|
|
||||||
|
common_prefixes: list[str] = []
|
||||||
|
filtered_objects: list = []
|
||||||
|
if delimiter:
|
||||||
|
seen_prefixes: set[str] = set()
|
||||||
|
for obj in objects:
|
||||||
|
key_after_prefix = obj.key[len(prefix):] if prefix else obj.key
|
||||||
|
if delimiter in key_after_prefix:
|
||||||
|
common_prefix = prefix + key_after_prefix.split(delimiter)[0] + delimiter
|
||||||
|
if common_prefix not in seen_prefixes:
|
||||||
|
seen_prefixes.add(common_prefix)
|
||||||
|
common_prefixes.append(common_prefix)
|
||||||
|
else:
|
||||||
|
filtered_objects.append(obj)
|
||||||
|
objects = filtered_objects
|
||||||
|
common_prefixes = sorted(common_prefixes)
|
||||||
|
|
||||||
|
total_items = len(objects) + len(common_prefixes)
|
||||||
|
is_truncated = total_items > max_keys or list_result.is_truncated
|
||||||
|
|
||||||
|
if len(objects) >= max_keys:
|
||||||
|
objects = objects[:max_keys]
|
||||||
|
common_prefixes = []
|
||||||
|
else:
|
||||||
|
remaining = max_keys - len(objects)
|
||||||
|
common_prefixes = common_prefixes[:remaining]
|
||||||
|
|
||||||
|
next_marker = ""
|
||||||
|
next_continuation_token = ""
|
||||||
|
if is_truncated:
|
||||||
|
if objects:
|
||||||
|
next_marker = objects[-1].key
|
||||||
|
elif common_prefixes:
|
||||||
|
next_marker = common_prefixes[-1].rstrip(delimiter) if delimiter else common_prefixes[-1]
|
||||||
|
|
||||||
|
if list_type == "2" and next_marker:
|
||||||
|
next_continuation_token = base64.urlsafe_b64encode(next_marker.encode()).decode("utf-8")
|
||||||
|
|
||||||
if list_type == "2":
|
if list_type == "2":
|
||||||
root = Element("ListBucketResult")
|
root = Element("ListBucketResult")
|
||||||
SubElement(root, "Name").text = bucket_name
|
SubElement(root, "Name").text = bucket_name
|
||||||
|
|||||||
@@ -245,7 +245,6 @@ def stream_objects_ndjson(
|
|||||||
url_templates: dict[str, str],
|
url_templates: dict[str, str],
|
||||||
display_tz: str = "UTC",
|
display_tz: str = "UTC",
|
||||||
versioning_enabled: bool = False,
|
versioning_enabled: bool = False,
|
||||||
delimiter: Optional[str] = None,
|
|
||||||
) -> Generator[str, None, None]:
|
) -> Generator[str, None, None]:
|
||||||
meta_line = json.dumps({
|
meta_line = json.dumps({
|
||||||
"type": "meta",
|
"type": "meta",
|
||||||
@@ -259,20 +258,11 @@ def stream_objects_ndjson(
|
|||||||
kwargs: dict[str, Any] = {"Bucket": bucket_name, "MaxKeys": 1000}
|
kwargs: dict[str, Any] = {"Bucket": bucket_name, "MaxKeys": 1000}
|
||||||
if prefix:
|
if prefix:
|
||||||
kwargs["Prefix"] = prefix
|
kwargs["Prefix"] = prefix
|
||||||
if delimiter:
|
|
||||||
kwargs["Delimiter"] = delimiter
|
|
||||||
|
|
||||||
running_count = 0
|
|
||||||
try:
|
try:
|
||||||
paginator = client.get_paginator("list_objects_v2")
|
paginator = client.get_paginator("list_objects_v2")
|
||||||
for page in paginator.paginate(**kwargs):
|
for page in paginator.paginate(**kwargs):
|
||||||
for cp in page.get("CommonPrefixes", []):
|
for obj in page.get("Contents", []):
|
||||||
yield json.dumps({
|
|
||||||
"type": "folder",
|
|
||||||
"prefix": cp["Prefix"],
|
|
||||||
}) + "\n"
|
|
||||||
page_contents = page.get("Contents", [])
|
|
||||||
for obj in page_contents:
|
|
||||||
last_mod = obj["LastModified"]
|
last_mod = obj["LastModified"]
|
||||||
yield json.dumps({
|
yield json.dumps({
|
||||||
"type": "object",
|
"type": "object",
|
||||||
@@ -283,8 +273,6 @@ def stream_objects_ndjson(
|
|||||||
"last_modified_iso": format_datetime_iso(last_mod, display_tz),
|
"last_modified_iso": format_datetime_iso(last_mod, display_tz),
|
||||||
"etag": obj.get("ETag", "").strip('"'),
|
"etag": obj.get("ETag", "").strip('"'),
|
||||||
}) + "\n"
|
}) + "\n"
|
||||||
running_count += len(page_contents)
|
|
||||||
yield json.dumps({"type": "count", "total_count": running_count}) + "\n"
|
|
||||||
except ClientError as exc:
|
except ClientError as exc:
|
||||||
error_msg = exc.response.get("Error", {}).get("Message", "S3 operation failed")
|
error_msg = exc.response.get("Error", {}).get("Message", "S3 operation failed")
|
||||||
yield json.dumps({"type": "error", "error": error_msg}) + "\n"
|
yield json.dumps({"type": "error", "error": error_msg}) + "\n"
|
||||||
|
|||||||
288
app/storage.py
288
app/storage.py
@@ -154,15 +154,6 @@ class ListObjectsResult:
|
|||||||
total_count: Optional[int] = None
|
total_count: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class ShallowListResult:
|
|
||||||
"""Result for delimiter-aware directory-level listing."""
|
|
||||||
objects: List[ObjectMeta]
|
|
||||||
common_prefixes: List[str]
|
|
||||||
is_truncated: bool
|
|
||||||
next_continuation_token: Optional[str]
|
|
||||||
|
|
||||||
|
|
||||||
def _utcnow() -> datetime:
|
def _utcnow() -> datetime:
|
||||||
return datetime.now(timezone.utc)
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
@@ -288,41 +279,25 @@ class ObjectStorage:
|
|||||||
version_count = 0
|
version_count = 0
|
||||||
version_bytes = 0
|
version_bytes = 0
|
||||||
|
|
||||||
internal = self.INTERNAL_FOLDERS
|
|
||||||
bucket_str = str(bucket_path)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
stack = [bucket_str]
|
for path in bucket_path.rglob("*"):
|
||||||
while stack:
|
if path.is_file():
|
||||||
current = stack.pop()
|
rel = path.relative_to(bucket_path)
|
||||||
try:
|
if not rel.parts:
|
||||||
with os.scandir(current) as it:
|
continue
|
||||||
for entry in it:
|
top_folder = rel.parts[0]
|
||||||
if current == bucket_str and entry.name in internal:
|
if top_folder not in self.INTERNAL_FOLDERS:
|
||||||
continue
|
stat = path.stat()
|
||||||
if entry.is_dir(follow_symlinks=False):
|
object_count += 1
|
||||||
stack.append(entry.path)
|
total_bytes += stat.st_size
|
||||||
elif entry.is_file(follow_symlinks=False):
|
|
||||||
object_count += 1
|
|
||||||
total_bytes += entry.stat(follow_symlinks=False).st_size
|
|
||||||
except PermissionError:
|
|
||||||
continue
|
|
||||||
|
|
||||||
versions_root = self._bucket_versions_root(bucket_name)
|
versions_root = self._bucket_versions_root(bucket_name)
|
||||||
if versions_root.exists():
|
if versions_root.exists():
|
||||||
v_stack = [str(versions_root)]
|
for path in versions_root.rglob("*.bin"):
|
||||||
while v_stack:
|
if path.is_file():
|
||||||
v_current = v_stack.pop()
|
stat = path.stat()
|
||||||
try:
|
version_count += 1
|
||||||
with os.scandir(v_current) as it:
|
version_bytes += stat.st_size
|
||||||
for entry in it:
|
|
||||||
if entry.is_dir(follow_symlinks=False):
|
|
||||||
v_stack.append(entry.path)
|
|
||||||
elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".bin"):
|
|
||||||
version_count += 1
|
|
||||||
version_bytes += entry.stat(follow_symlinks=False).st_size
|
|
||||||
except PermissionError:
|
|
||||||
continue
|
|
||||||
except OSError:
|
except OSError:
|
||||||
if cached_stats is not None:
|
if cached_stats is not None:
|
||||||
return cached_stats
|
return cached_stats
|
||||||
@@ -402,18 +377,9 @@ class ObjectStorage:
|
|||||||
raise StorageError("Bucket contains archived object versions")
|
raise StorageError("Bucket contains archived object versions")
|
||||||
if has_multipart:
|
if has_multipart:
|
||||||
raise StorageError("Bucket has active multipart uploads")
|
raise StorageError("Bucket has active multipart uploads")
|
||||||
bucket_id = bucket_path.name
|
|
||||||
self._remove_tree(bucket_path)
|
self._remove_tree(bucket_path)
|
||||||
self._remove_tree(self._system_bucket_root(bucket_id))
|
self._remove_tree(self._system_bucket_root(bucket_path.name))
|
||||||
self._remove_tree(self._multipart_bucket_root(bucket_id))
|
self._remove_tree(self._multipart_bucket_root(bucket_path.name))
|
||||||
self._bucket_config_cache.pop(bucket_id, None)
|
|
||||||
with self._cache_lock:
|
|
||||||
self._object_cache.pop(bucket_id, None)
|
|
||||||
self._cache_version.pop(bucket_id, None)
|
|
||||||
self._sorted_key_cache.pop(bucket_id, None)
|
|
||||||
stale = [k for k in self._meta_read_cache if k[0] == bucket_id]
|
|
||||||
for k in stale:
|
|
||||||
del self._meta_read_cache[k]
|
|
||||||
|
|
||||||
def list_objects(
|
def list_objects(
|
||||||
self,
|
self,
|
||||||
@@ -496,202 +462,6 @@ class ObjectStorage:
|
|||||||
result = self.list_objects(bucket_name, max_keys=100000)
|
result = self.list_objects(bucket_name, max_keys=100000)
|
||||||
return result.objects
|
return result.objects
|
||||||
|
|
||||||
def list_objects_shallow(
|
|
||||||
self,
|
|
||||||
bucket_name: str,
|
|
||||||
*,
|
|
||||||
prefix: str = "",
|
|
||||||
delimiter: str = "/",
|
|
||||||
max_keys: int = 1000,
|
|
||||||
continuation_token: Optional[str] = None,
|
|
||||||
) -> ShallowListResult:
|
|
||||||
import bisect
|
|
||||||
|
|
||||||
bucket_path = self._bucket_path(bucket_name)
|
|
||||||
if not bucket_path.exists():
|
|
||||||
raise BucketNotFoundError("Bucket does not exist")
|
|
||||||
bucket_id = bucket_path.name
|
|
||||||
|
|
||||||
if delimiter != "/" or (prefix and not prefix.endswith(delimiter)):
|
|
||||||
return self._shallow_via_full_scan(
|
|
||||||
bucket_name, prefix=prefix, delimiter=delimiter,
|
|
||||||
max_keys=max_keys, continuation_token=continuation_token,
|
|
||||||
)
|
|
||||||
|
|
||||||
target_dir = bucket_path
|
|
||||||
if prefix:
|
|
||||||
safe_prefix_path = Path(prefix.rstrip("/"))
|
|
||||||
if ".." in safe_prefix_path.parts:
|
|
||||||
return ShallowListResult(
|
|
||||||
objects=[], common_prefixes=[],
|
|
||||||
is_truncated=False, next_continuation_token=None,
|
|
||||||
)
|
|
||||||
target_dir = bucket_path / safe_prefix_path
|
|
||||||
try:
|
|
||||||
resolved = target_dir.resolve()
|
|
||||||
bucket_resolved = bucket_path.resolve()
|
|
||||||
if not str(resolved).startswith(str(bucket_resolved) + os.sep) and resolved != bucket_resolved:
|
|
||||||
return ShallowListResult(
|
|
||||||
objects=[], common_prefixes=[],
|
|
||||||
is_truncated=False, next_continuation_token=None,
|
|
||||||
)
|
|
||||||
except (OSError, ValueError):
|
|
||||||
return ShallowListResult(
|
|
||||||
objects=[], common_prefixes=[],
|
|
||||||
is_truncated=False, next_continuation_token=None,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not target_dir.exists() or not target_dir.is_dir():
|
|
||||||
return ShallowListResult(
|
|
||||||
objects=[], common_prefixes=[],
|
|
||||||
is_truncated=False, next_continuation_token=None,
|
|
||||||
)
|
|
||||||
|
|
||||||
etag_index_path = self._system_bucket_root(bucket_id) / "etag_index.json"
|
|
||||||
meta_cache: Dict[str, str] = {}
|
|
||||||
if etag_index_path.exists():
|
|
||||||
try:
|
|
||||||
with open(etag_index_path, 'r', encoding='utf-8') as f:
|
|
||||||
meta_cache = json.load(f)
|
|
||||||
except (OSError, json.JSONDecodeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
entries_files: list[tuple[str, int, float, Optional[str]]] = []
|
|
||||||
entries_dirs: list[str] = []
|
|
||||||
|
|
||||||
try:
|
|
||||||
with os.scandir(str(target_dir)) as it:
|
|
||||||
for entry in it:
|
|
||||||
name = entry.name
|
|
||||||
if name in self.INTERNAL_FOLDERS:
|
|
||||||
continue
|
|
||||||
if entry.is_dir(follow_symlinks=False):
|
|
||||||
cp = prefix + name + delimiter
|
|
||||||
entries_dirs.append(cp)
|
|
||||||
elif entry.is_file(follow_symlinks=False):
|
|
||||||
key = prefix + name
|
|
||||||
try:
|
|
||||||
st = entry.stat()
|
|
||||||
etag = meta_cache.get(key)
|
|
||||||
entries_files.append((key, st.st_size, st.st_mtime, etag))
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
except OSError:
|
|
||||||
return ShallowListResult(
|
|
||||||
objects=[], common_prefixes=[],
|
|
||||||
is_truncated=False, next_continuation_token=None,
|
|
||||||
)
|
|
||||||
|
|
||||||
entries_dirs.sort()
|
|
||||||
entries_files.sort(key=lambda x: x[0])
|
|
||||||
|
|
||||||
all_items: list[tuple[str, bool]] = []
|
|
||||||
fi, di = 0, 0
|
|
||||||
while fi < len(entries_files) and di < len(entries_dirs):
|
|
||||||
if entries_files[fi][0] <= entries_dirs[di]:
|
|
||||||
all_items.append((entries_files[fi][0], False))
|
|
||||||
fi += 1
|
|
||||||
else:
|
|
||||||
all_items.append((entries_dirs[di], True))
|
|
||||||
di += 1
|
|
||||||
while fi < len(entries_files):
|
|
||||||
all_items.append((entries_files[fi][0], False))
|
|
||||||
fi += 1
|
|
||||||
while di < len(entries_dirs):
|
|
||||||
all_items.append((entries_dirs[di], True))
|
|
||||||
di += 1
|
|
||||||
|
|
||||||
files_map = {e[0]: e for e in entries_files}
|
|
||||||
|
|
||||||
start_index = 0
|
|
||||||
if continuation_token:
|
|
||||||
all_keys = [item[0] for item in all_items]
|
|
||||||
start_index = bisect.bisect_right(all_keys, continuation_token)
|
|
||||||
|
|
||||||
selected = all_items[start_index:start_index + max_keys]
|
|
||||||
is_truncated = (start_index + max_keys) < len(all_items)
|
|
||||||
|
|
||||||
result_objects: list[ObjectMeta] = []
|
|
||||||
result_prefixes: list[str] = []
|
|
||||||
for item_key, is_dir in selected:
|
|
||||||
if is_dir:
|
|
||||||
result_prefixes.append(item_key)
|
|
||||||
else:
|
|
||||||
fdata = files_map[item_key]
|
|
||||||
result_objects.append(ObjectMeta(
|
|
||||||
key=fdata[0],
|
|
||||||
size=fdata[1],
|
|
||||||
last_modified=datetime.fromtimestamp(fdata[2], timezone.utc),
|
|
||||||
etag=fdata[3],
|
|
||||||
metadata=None,
|
|
||||||
))
|
|
||||||
|
|
||||||
next_token = None
|
|
||||||
if is_truncated and selected:
|
|
||||||
next_token = selected[-1][0]
|
|
||||||
|
|
||||||
return ShallowListResult(
|
|
||||||
objects=result_objects,
|
|
||||||
common_prefixes=result_prefixes,
|
|
||||||
is_truncated=is_truncated,
|
|
||||||
next_continuation_token=next_token,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _shallow_via_full_scan(
|
|
||||||
self,
|
|
||||||
bucket_name: str,
|
|
||||||
*,
|
|
||||||
prefix: str = "",
|
|
||||||
delimiter: str = "/",
|
|
||||||
max_keys: int = 1000,
|
|
||||||
continuation_token: Optional[str] = None,
|
|
||||||
) -> ShallowListResult:
|
|
||||||
list_result = self.list_objects(
|
|
||||||
bucket_name,
|
|
||||||
max_keys=max_keys * 10,
|
|
||||||
continuation_token=continuation_token,
|
|
||||||
prefix=prefix or None,
|
|
||||||
)
|
|
||||||
|
|
||||||
common_prefixes: list[str] = []
|
|
||||||
filtered_objects: list[ObjectMeta] = []
|
|
||||||
seen_prefixes: set[str] = set()
|
|
||||||
|
|
||||||
for obj in list_result.objects:
|
|
||||||
key_after_prefix = obj.key[len(prefix):] if prefix else obj.key
|
|
||||||
if delimiter in key_after_prefix:
|
|
||||||
cp = prefix + key_after_prefix.split(delimiter)[0] + delimiter
|
|
||||||
if cp not in seen_prefixes:
|
|
||||||
seen_prefixes.add(cp)
|
|
||||||
common_prefixes.append(cp)
|
|
||||||
else:
|
|
||||||
filtered_objects.append(obj)
|
|
||||||
|
|
||||||
common_prefixes.sort()
|
|
||||||
total_items = len(filtered_objects) + len(common_prefixes)
|
|
||||||
is_truncated = total_items > max_keys or list_result.is_truncated
|
|
||||||
|
|
||||||
if len(filtered_objects) >= max_keys:
|
|
||||||
filtered_objects = filtered_objects[:max_keys]
|
|
||||||
common_prefixes = []
|
|
||||||
else:
|
|
||||||
remaining = max_keys - len(filtered_objects)
|
|
||||||
common_prefixes = common_prefixes[:remaining]
|
|
||||||
|
|
||||||
next_token = None
|
|
||||||
if is_truncated:
|
|
||||||
if filtered_objects:
|
|
||||||
next_token = filtered_objects[-1].key
|
|
||||||
elif common_prefixes:
|
|
||||||
next_token = common_prefixes[-1].rstrip(delimiter) if delimiter else common_prefixes[-1]
|
|
||||||
|
|
||||||
return ShallowListResult(
|
|
||||||
objects=filtered_objects,
|
|
||||||
common_prefixes=common_prefixes,
|
|
||||||
is_truncated=is_truncated,
|
|
||||||
next_continuation_token=next_token,
|
|
||||||
)
|
|
||||||
|
|
||||||
def put_object(
|
def put_object(
|
||||||
self,
|
self,
|
||||||
bucket_name: str,
|
bucket_name: str,
|
||||||
@@ -2062,40 +1832,30 @@ class ObjectStorage:
|
|||||||
|
|
||||||
def _read_bucket_config(self, bucket_name: str) -> dict[str, Any]:
|
def _read_bucket_config(self, bucket_name: str) -> dict[str, Any]:
|
||||||
now = time.time()
|
now = time.time()
|
||||||
config_path = self._bucket_config_path(bucket_name)
|
|
||||||
cached = self._bucket_config_cache.get(bucket_name)
|
cached = self._bucket_config_cache.get(bucket_name)
|
||||||
if cached:
|
if cached:
|
||||||
config, cached_time, cached_mtime = cached
|
config, cached_time = cached
|
||||||
if now - cached_time < self._bucket_config_cache_ttl:
|
if now - cached_time < self._bucket_config_cache_ttl:
|
||||||
try:
|
return config.copy()
|
||||||
current_mtime = config_path.stat().st_mtime if config_path.exists() else 0.0
|
|
||||||
except OSError:
|
|
||||||
current_mtime = 0.0
|
|
||||||
if current_mtime == cached_mtime:
|
|
||||||
return config.copy()
|
|
||||||
|
|
||||||
|
config_path = self._bucket_config_path(bucket_name)
|
||||||
if not config_path.exists():
|
if not config_path.exists():
|
||||||
self._bucket_config_cache[bucket_name] = ({}, now, 0.0)
|
self._bucket_config_cache[bucket_name] = ({}, now)
|
||||||
return {}
|
return {}
|
||||||
try:
|
try:
|
||||||
data = json.loads(config_path.read_text(encoding="utf-8"))
|
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||||
config = data if isinstance(data, dict) else {}
|
config = data if isinstance(data, dict) else {}
|
||||||
mtime = config_path.stat().st_mtime
|
self._bucket_config_cache[bucket_name] = (config, now)
|
||||||
self._bucket_config_cache[bucket_name] = (config, now, mtime)
|
|
||||||
return config.copy()
|
return config.copy()
|
||||||
except (OSError, json.JSONDecodeError):
|
except (OSError, json.JSONDecodeError):
|
||||||
self._bucket_config_cache[bucket_name] = ({}, now, 0.0)
|
self._bucket_config_cache[bucket_name] = ({}, now)
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
def _write_bucket_config(self, bucket_name: str, payload: dict[str, Any]) -> None:
|
def _write_bucket_config(self, bucket_name: str, payload: dict[str, Any]) -> None:
|
||||||
config_path = self._bucket_config_path(bucket_name)
|
config_path = self._bucket_config_path(bucket_name)
|
||||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
config_path.write_text(json.dumps(payload), encoding="utf-8")
|
config_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||||
try:
|
self._bucket_config_cache[bucket_name] = (payload.copy(), time.time())
|
||||||
mtime = config_path.stat().st_mtime
|
|
||||||
except OSError:
|
|
||||||
mtime = 0.0
|
|
||||||
self._bucket_config_cache[bucket_name] = (payload.copy(), time.time(), mtime)
|
|
||||||
|
|
||||||
def _set_bucket_config_entry(self, bucket_name: str, key: str, value: Any | None) -> None:
|
def _set_bucket_config_entry(self, bucket_name: str, key: str, value: Any | None) -> None:
|
||||||
config = self._read_bucket_config(bucket_name)
|
config = self._read_bucket_config(bucket_name)
|
||||||
|
|||||||
@@ -616,7 +616,6 @@ def stream_bucket_objects(bucket_name: str):
|
|||||||
return jsonify({"error": str(exc)}), 403
|
return jsonify({"error": str(exc)}), 403
|
||||||
|
|
||||||
prefix = request.args.get("prefix") or None
|
prefix = request.args.get("prefix") or None
|
||||||
delimiter = request.args.get("delimiter") or None
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
client = get_session_s3_client()
|
client = get_session_s3_client()
|
||||||
@@ -630,7 +629,6 @@ def stream_bucket_objects(bucket_name: str):
|
|||||||
return Response(
|
return Response(
|
||||||
stream_objects_ndjson(
|
stream_objects_ndjson(
|
||||||
client, bucket_name, prefix, url_templates, display_tz, versioning_enabled,
|
client, bucket_name, prefix, url_templates, display_tz, versioning_enabled,
|
||||||
delimiter=delimiter,
|
|
||||||
),
|
),
|
||||||
mimetype='application/x-ndjson',
|
mimetype='application/x-ndjson',
|
||||||
headers={
|
headers={
|
||||||
@@ -1303,14 +1301,12 @@ def object_versions(bucket_name: str, object_key: str):
|
|||||||
for v in resp.get("Versions", []):
|
for v in resp.get("Versions", []):
|
||||||
if v.get("Key") != object_key:
|
if v.get("Key") != object_key:
|
||||||
continue
|
continue
|
||||||
if v.get("IsLatest", False):
|
|
||||||
continue
|
|
||||||
versions.append({
|
versions.append({
|
||||||
"version_id": v.get("VersionId", ""),
|
"version_id": v.get("VersionId", ""),
|
||||||
"last_modified": v["LastModified"].isoformat() if v.get("LastModified") else None,
|
"last_modified": v["LastModified"].isoformat() if v.get("LastModified") else None,
|
||||||
"size": v.get("Size", 0),
|
"size": v.get("Size", 0),
|
||||||
"etag": v.get("ETag", "").strip('"'),
|
"etag": v.get("ETag", "").strip('"'),
|
||||||
"is_latest": False,
|
"is_latest": v.get("IsLatest", False),
|
||||||
})
|
})
|
||||||
return jsonify({"versions": versions})
|
return jsonify({"versions": versions})
|
||||||
except (ClientError, EndpointConnectionError, ConnectionClosedError) as exc:
|
except (ClientError, EndpointConnectionError, ConnectionClosedError) as exc:
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
APP_VERSION = "0.3.1"
|
APP_VERSION = "0.3.0"
|
||||||
|
|
||||||
|
|
||||||
def get_version() -> str:
|
def get_version() -> str:
|
||||||
|
|||||||
@@ -137,11 +137,11 @@
|
|||||||
const versionPanel = document.getElementById('version-panel');
|
const versionPanel = document.getElementById('version-panel');
|
||||||
const versionList = document.getElementById('version-list');
|
const versionList = document.getElementById('version-list');
|
||||||
const refreshVersionsButton = document.getElementById('refreshVersionsButton');
|
const refreshVersionsButton = document.getElementById('refreshVersionsButton');
|
||||||
let archivedCard = document.getElementById('archived-objects-card');
|
const archivedCard = document.getElementById('archived-objects-card');
|
||||||
let archivedBody = archivedCard?.querySelector('[data-archived-body]');
|
const archivedBody = archivedCard?.querySelector('[data-archived-body]');
|
||||||
let archivedCountBadge = archivedCard?.querySelector('[data-archived-count]');
|
const archivedCountBadge = archivedCard?.querySelector('[data-archived-count]');
|
||||||
let archivedRefreshButton = archivedCard?.querySelector('[data-archived-refresh]');
|
const archivedRefreshButton = archivedCard?.querySelector('[data-archived-refresh]');
|
||||||
let archivedEndpoint = archivedCard?.dataset.archivedEndpoint;
|
const archivedEndpoint = archivedCard?.dataset.archivedEndpoint;
|
||||||
let versioningEnabled = objectsContainer?.dataset.versioning === 'true';
|
let versioningEnabled = objectsContainer?.dataset.versioning === 'true';
|
||||||
const versionsCache = new Map();
|
const versionsCache = new Map();
|
||||||
let activeRow = null;
|
let activeRow = null;
|
||||||
@@ -167,8 +167,6 @@
|
|||||||
let pageSize = 5000;
|
let pageSize = 5000;
|
||||||
let currentPrefix = '';
|
let currentPrefix = '';
|
||||||
let allObjects = [];
|
let allObjects = [];
|
||||||
let streamFolders = [];
|
|
||||||
let useDelimiterMode = true;
|
|
||||||
let urlTemplates = null;
|
let urlTemplates = null;
|
||||||
let streamAbortController = null;
|
let streamAbortController = null;
|
||||||
let useStreaming = !!objectsStreamUrl;
|
let useStreaming = !!objectsStreamUrl;
|
||||||
@@ -188,7 +186,7 @@
|
|||||||
let renderedRange = { start: 0, end: 0 };
|
let renderedRange = { start: 0, end: 0 };
|
||||||
|
|
||||||
let memoizedVisibleItems = null;
|
let memoizedVisibleItems = null;
|
||||||
let memoizedInputs = { objectCount: -1, folderCount: -1, prefix: null, filterTerm: null };
|
let memoizedInputs = { objectCount: -1, prefix: null, filterTerm: null };
|
||||||
|
|
||||||
const createObjectRow = (obj, displayKey = null) => {
|
const createObjectRow = (obj, displayKey = null) => {
|
||||||
const tr = document.createElement('tr');
|
const tr = document.createElement('tr');
|
||||||
@@ -321,13 +319,10 @@
|
|||||||
`;
|
`;
|
||||||
};
|
};
|
||||||
|
|
||||||
const bucketTotalObjects = objectsContainer ? parseInt(objectsContainer.dataset.bucketTotalObjects || '0', 10) : 0;
|
|
||||||
|
|
||||||
const updateObjectCountBadge = () => {
|
const updateObjectCountBadge = () => {
|
||||||
if (!objectCountBadge) return;
|
if (!objectCountBadge) return;
|
||||||
if (useDelimiterMode) {
|
if (totalObjectCount === 0) {
|
||||||
const total = bucketTotalObjects || totalObjectCount;
|
objectCountBadge.textContent = '0 objects';
|
||||||
objectCountBadge.textContent = `${total.toLocaleString()} object${total !== 1 ? 's' : ''}`;
|
|
||||||
} else {
|
} else {
|
||||||
objectCountBadge.textContent = `${totalObjectCount.toLocaleString()} object${totalObjectCount !== 1 ? 's' : ''}`;
|
objectCountBadge.textContent = `${totalObjectCount.toLocaleString()} object${totalObjectCount !== 1 ? 's' : ''}`;
|
||||||
}
|
}
|
||||||
@@ -354,7 +349,6 @@
|
|||||||
const computeVisibleItems = (forceRecompute = false) => {
|
const computeVisibleItems = (forceRecompute = false) => {
|
||||||
const currentInputs = {
|
const currentInputs = {
|
||||||
objectCount: allObjects.length,
|
objectCount: allObjects.length,
|
||||||
folderCount: streamFolders.length,
|
|
||||||
prefix: currentPrefix,
|
prefix: currentPrefix,
|
||||||
filterTerm: currentFilterTerm,
|
filterTerm: currentFilterTerm,
|
||||||
sortField: currentSortField,
|
sortField: currentSortField,
|
||||||
@@ -364,7 +358,6 @@
|
|||||||
if (!forceRecompute &&
|
if (!forceRecompute &&
|
||||||
memoizedVisibleItems !== null &&
|
memoizedVisibleItems !== null &&
|
||||||
memoizedInputs.objectCount === currentInputs.objectCount &&
|
memoizedInputs.objectCount === currentInputs.objectCount &&
|
||||||
memoizedInputs.folderCount === currentInputs.folderCount &&
|
|
||||||
memoizedInputs.prefix === currentInputs.prefix &&
|
memoizedInputs.prefix === currentInputs.prefix &&
|
||||||
memoizedInputs.filterTerm === currentInputs.filterTerm &&
|
memoizedInputs.filterTerm === currentInputs.filterTerm &&
|
||||||
memoizedInputs.sortField === currentInputs.sortField &&
|
memoizedInputs.sortField === currentInputs.sortField &&
|
||||||
@@ -373,53 +366,36 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
const items = [];
|
const items = [];
|
||||||
|
const folders = new Set();
|
||||||
|
|
||||||
if (useDelimiterMode && streamFolders.length > 0) {
|
allObjects.forEach(obj => {
|
||||||
streamFolders.forEach(folderPath => {
|
if (!obj.key.startsWith(currentPrefix)) return;
|
||||||
const folderName = folderPath.slice(currentPrefix.length).replace(/\/$/, '');
|
|
||||||
if (!currentFilterTerm || folderName.toLowerCase().includes(currentFilterTerm)) {
|
const remainder = obj.key.slice(currentPrefix.length);
|
||||||
items.push({ type: 'folder', path: folderPath, displayKey: folderName });
|
|
||||||
}
|
if (!remainder) return;
|
||||||
});
|
|
||||||
allObjects.forEach(obj => {
|
const isFolderMarker = obj.key.endsWith('/') && obj.size === 0;
|
||||||
const remainder = obj.key.slice(currentPrefix.length);
|
const slashIndex = remainder.indexOf('/');
|
||||||
if (!remainder) return;
|
|
||||||
|
if (slashIndex === -1 && !isFolderMarker) {
|
||||||
if (!currentFilterTerm || remainder.toLowerCase().includes(currentFilterTerm)) {
|
if (!currentFilterTerm || remainder.toLowerCase().includes(currentFilterTerm)) {
|
||||||
items.push({ type: 'file', data: obj, displayKey: remainder });
|
items.push({ type: 'file', data: obj, displayKey: remainder });
|
||||||
}
|
}
|
||||||
});
|
} else {
|
||||||
} else {
|
const effectiveSlashIndex = isFolderMarker && slashIndex === remainder.length - 1
|
||||||
const folders = new Set();
|
? slashIndex
|
||||||
|
: (slashIndex === -1 ? remainder.length - 1 : slashIndex);
|
||||||
allObjects.forEach(obj => {
|
const folderName = remainder.slice(0, effectiveSlashIndex);
|
||||||
if (!obj.key.startsWith(currentPrefix)) return;
|
const folderPath = currentPrefix + folderName + '/';
|
||||||
|
if (!folders.has(folderPath)) {
|
||||||
const remainder = obj.key.slice(currentPrefix.length);
|
folders.add(folderPath);
|
||||||
|
if (!currentFilterTerm || folderName.toLowerCase().includes(currentFilterTerm)) {
|
||||||
if (!remainder) return;
|
items.push({ type: 'folder', path: folderPath, displayKey: folderName });
|
||||||
|
|
||||||
const isFolderMarker = obj.key.endsWith('/') && obj.size === 0;
|
|
||||||
const slashIndex = remainder.indexOf('/');
|
|
||||||
|
|
||||||
if (slashIndex === -1 && !isFolderMarker) {
|
|
||||||
if (!currentFilterTerm || remainder.toLowerCase().includes(currentFilterTerm)) {
|
|
||||||
items.push({ type: 'file', data: obj, displayKey: remainder });
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
const effectiveSlashIndex = isFolderMarker && slashIndex === remainder.length - 1
|
|
||||||
? slashIndex
|
|
||||||
: (slashIndex === -1 ? remainder.length - 1 : slashIndex);
|
|
||||||
const folderName = remainder.slice(0, effectiveSlashIndex);
|
|
||||||
const folderPath = currentPrefix + folderName + '/';
|
|
||||||
if (!folders.has(folderPath)) {
|
|
||||||
folders.add(folderPath);
|
|
||||||
if (!currentFilterTerm || folderName.toLowerCase().includes(currentFilterTerm)) {
|
|
||||||
items.push({ type: 'folder', path: folderPath, displayKey: folderName });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
}
|
});
|
||||||
|
|
||||||
items.sort((a, b) => {
|
items.sort((a, b) => {
|
||||||
if (a.type === 'folder' && b.type === 'file') return -1;
|
if (a.type === 'folder' && b.type === 'file') return -1;
|
||||||
@@ -495,7 +471,7 @@
|
|||||||
renderedRange = { start: -1, end: -1 };
|
renderedRange = { start: -1, end: -1 };
|
||||||
|
|
||||||
if (visibleItems.length === 0) {
|
if (visibleItems.length === 0) {
|
||||||
if (allObjects.length === 0 && streamFolders.length === 0 && !hasMoreObjects) {
|
if (allObjects.length === 0 && !hasMoreObjects) {
|
||||||
showEmptyState();
|
showEmptyState();
|
||||||
} else {
|
} else {
|
||||||
objectsTableBody.innerHTML = `
|
objectsTableBody.innerHTML = `
|
||||||
@@ -524,7 +500,15 @@
|
|||||||
const updateFolderViewStatus = () => {
|
const updateFolderViewStatus = () => {
|
||||||
const folderViewStatusEl = document.getElementById('folder-view-status');
|
const folderViewStatusEl = document.getElementById('folder-view-status');
|
||||||
if (!folderViewStatusEl) return;
|
if (!folderViewStatusEl) return;
|
||||||
folderViewStatusEl.classList.add('d-none');
|
|
||||||
|
if (currentPrefix) {
|
||||||
|
const folderCount = visibleItems.filter(i => i.type === 'folder').length;
|
||||||
|
const fileCount = visibleItems.filter(i => i.type === 'file').length;
|
||||||
|
folderViewStatusEl.innerHTML = `<span class="text-muted">${folderCount} folder${folderCount !== 1 ? 's' : ''}, ${fileCount} file${fileCount !== 1 ? 's' : ''} in this view</span>`;
|
||||||
|
folderViewStatusEl.classList.remove('d-none');
|
||||||
|
} else {
|
||||||
|
folderViewStatusEl.classList.add('d-none');
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const processStreamObject = (obj) => {
|
const processStreamObject = (obj) => {
|
||||||
@@ -552,30 +536,21 @@
|
|||||||
let lastStreamRenderTime = 0;
|
let lastStreamRenderTime = 0;
|
||||||
const STREAM_RENDER_THROTTLE_MS = 500;
|
const STREAM_RENDER_THROTTLE_MS = 500;
|
||||||
|
|
||||||
const buildBottomStatusText = (complete) => {
|
|
||||||
if (!complete) {
|
|
||||||
const countText = totalObjectCount > 0 ? ` of ${totalObjectCount.toLocaleString()}` : '';
|
|
||||||
return `${loadedObjectCount.toLocaleString()}${countText} loading...`;
|
|
||||||
}
|
|
||||||
const parts = [];
|
|
||||||
if (useDelimiterMode && streamFolders.length > 0) {
|
|
||||||
parts.push(`${streamFolders.length.toLocaleString()} folder${streamFolders.length !== 1 ? 's' : ''}`);
|
|
||||||
}
|
|
||||||
parts.push(`${loadedObjectCount.toLocaleString()} object${loadedObjectCount !== 1 ? 's' : ''}`);
|
|
||||||
return parts.join(', ');
|
|
||||||
};
|
|
||||||
|
|
||||||
const flushPendingStreamObjects = () => {
|
const flushPendingStreamObjects = () => {
|
||||||
if (pendingStreamObjects.length > 0) {
|
if (pendingStreamObjects.length === 0) return;
|
||||||
const batch = pendingStreamObjects.splice(0, pendingStreamObjects.length);
|
const batch = pendingStreamObjects.splice(0, pendingStreamObjects.length);
|
||||||
batch.forEach(obj => {
|
batch.forEach(obj => {
|
||||||
loadedObjectCount++;
|
loadedObjectCount++;
|
||||||
allObjects.push(obj);
|
allObjects.push(obj);
|
||||||
});
|
});
|
||||||
}
|
|
||||||
updateObjectCountBadge();
|
updateObjectCountBadge();
|
||||||
if (loadMoreStatus) {
|
if (loadMoreStatus) {
|
||||||
loadMoreStatus.textContent = buildBottomStatusText(streamingComplete);
|
if (streamingComplete) {
|
||||||
|
loadMoreStatus.textContent = `${loadedObjectCount.toLocaleString()} objects`;
|
||||||
|
} else {
|
||||||
|
const countText = totalObjectCount > 0 ? ` of ${totalObjectCount.toLocaleString()}` : '';
|
||||||
|
loadMoreStatus.textContent = `${loadedObjectCount.toLocaleString()}${countText} loading...`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (objectsLoadingRow && objectsLoadingRow.parentNode) {
|
if (objectsLoadingRow && objectsLoadingRow.parentNode) {
|
||||||
const loadingText = objectsLoadingRow.querySelector('p');
|
const loadingText = objectsLoadingRow.querySelector('p');
|
||||||
@@ -610,9 +585,8 @@
|
|||||||
loadedObjectCount = 0;
|
loadedObjectCount = 0;
|
||||||
totalObjectCount = 0;
|
totalObjectCount = 0;
|
||||||
allObjects = [];
|
allObjects = [];
|
||||||
streamFolders = [];
|
|
||||||
memoizedVisibleItems = null;
|
memoizedVisibleItems = null;
|
||||||
memoizedInputs = { objectCount: -1, folderCount: -1, prefix: null, filterTerm: null };
|
memoizedInputs = { objectCount: -1, prefix: null, filterTerm: null };
|
||||||
pendingStreamObjects = [];
|
pendingStreamObjects = [];
|
||||||
lastStreamRenderTime = 0;
|
lastStreamRenderTime = 0;
|
||||||
|
|
||||||
@@ -621,7 +595,6 @@
|
|||||||
try {
|
try {
|
||||||
const params = new URLSearchParams();
|
const params = new URLSearchParams();
|
||||||
if (currentPrefix) params.set('prefix', currentPrefix);
|
if (currentPrefix) params.set('prefix', currentPrefix);
|
||||||
if (useDelimiterMode) params.set('delimiter', '/');
|
|
||||||
|
|
||||||
const response = await fetch(`${objectsStreamUrl}?${params}`, {
|
const response = await fetch(`${objectsStreamUrl}?${params}`, {
|
||||||
signal: streamAbortController.signal
|
signal: streamAbortController.signal
|
||||||
@@ -666,10 +639,6 @@
|
|||||||
if (loadingText) loadingText.textContent = `Loading 0 of ${totalObjectCount.toLocaleString()} objects...`;
|
if (loadingText) loadingText.textContent = `Loading 0 of ${totalObjectCount.toLocaleString()} objects...`;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case 'folder':
|
|
||||||
streamFolders.push(msg.prefix);
|
|
||||||
scheduleStreamRender();
|
|
||||||
break;
|
|
||||||
case 'object':
|
case 'object':
|
||||||
pendingStreamObjects.push(processStreamObject(msg));
|
pendingStreamObjects.push(processStreamObject(msg));
|
||||||
if (pendingStreamObjects.length >= STREAM_RENDER_BATCH) {
|
if (pendingStreamObjects.length >= STREAM_RENDER_BATCH) {
|
||||||
@@ -713,7 +682,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (loadMoreStatus) {
|
if (loadMoreStatus) {
|
||||||
loadMoreStatus.textContent = buildBottomStatusText(true);
|
loadMoreStatus.textContent = `${loadedObjectCount.toLocaleString()} objects`;
|
||||||
}
|
}
|
||||||
refreshVirtualList();
|
refreshVirtualList();
|
||||||
renderBreadcrumb(currentPrefix);
|
renderBreadcrumb(currentPrefix);
|
||||||
@@ -741,9 +710,8 @@
|
|||||||
loadedObjectCount = 0;
|
loadedObjectCount = 0;
|
||||||
totalObjectCount = 0;
|
totalObjectCount = 0;
|
||||||
allObjects = [];
|
allObjects = [];
|
||||||
streamFolders = [];
|
|
||||||
memoizedVisibleItems = null;
|
memoizedVisibleItems = null;
|
||||||
memoizedInputs = { objectCount: -1, folderCount: -1, prefix: null, filterTerm: null };
|
memoizedInputs = { objectCount: -1, prefix: null, filterTerm: null };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (append && loadMoreSpinner) {
|
if (append && loadMoreSpinner) {
|
||||||
@@ -945,7 +913,7 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const hasFolders = () => streamFolders.length > 0 || allObjects.some(obj => obj.key.includes('/'));
|
const hasFolders = () => allObjects.some(obj => obj.key.includes('/'));
|
||||||
|
|
||||||
const getFoldersAtPrefix = (prefix) => {
|
const getFoldersAtPrefix = (prefix) => {
|
||||||
const folders = new Set();
|
const folders = new Set();
|
||||||
@@ -972,9 +940,6 @@
|
|||||||
};
|
};
|
||||||
|
|
||||||
const countObjectsInFolder = (folderPrefix) => {
|
const countObjectsInFolder = (folderPrefix) => {
|
||||||
if (useDelimiterMode) {
|
|
||||||
return { count: 0, mayHaveMore: true };
|
|
||||||
}
|
|
||||||
const count = allObjects.filter(obj => obj.key.startsWith(folderPrefix)).length;
|
const count = allObjects.filter(obj => obj.key.startsWith(folderPrefix)).length;
|
||||||
return { count, mayHaveMore: hasMoreObjects };
|
return { count, mayHaveMore: hasMoreObjects };
|
||||||
};
|
};
|
||||||
@@ -1053,13 +1018,7 @@
|
|||||||
const createFolderRow = (folderPath, displayName = null) => {
|
const createFolderRow = (folderPath, displayName = null) => {
|
||||||
const folderName = displayName || folderPath.slice(currentPrefix.length).replace(/\/$/, '');
|
const folderName = displayName || folderPath.slice(currentPrefix.length).replace(/\/$/, '');
|
||||||
const { count: objectCount, mayHaveMore } = countObjectsInFolder(folderPath);
|
const { count: objectCount, mayHaveMore } = countObjectsInFolder(folderPath);
|
||||||
let countLine = '';
|
const countDisplay = mayHaveMore ? `${objectCount}+` : objectCount;
|
||||||
if (useDelimiterMode) {
|
|
||||||
countLine = '';
|
|
||||||
} else {
|
|
||||||
const countDisplay = mayHaveMore ? `${objectCount}+` : objectCount;
|
|
||||||
countLine = `<div class="text-muted small ms-4 ps-2">${countDisplay} object${objectCount !== 1 ? 's' : ''}</div>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const tr = document.createElement('tr');
|
const tr = document.createElement('tr');
|
||||||
tr.className = 'folder-row';
|
tr.className = 'folder-row';
|
||||||
@@ -1077,7 +1036,7 @@
|
|||||||
</svg>
|
</svg>
|
||||||
<span>${escapeHtml(folderName)}/</span>
|
<span>${escapeHtml(folderName)}/</span>
|
||||||
</div>
|
</div>
|
||||||
${countLine}
|
<div class="text-muted small ms-4 ps-2">${countDisplay} object${objectCount !== 1 ? 's' : ''}</div>
|
||||||
</td>
|
</td>
|
||||||
<td class="text-end text-nowrap">
|
<td class="text-end text-nowrap">
|
||||||
<span class="text-muted small">—</span>
|
<span class="text-muted small">—</span>
|
||||||
@@ -1578,7 +1537,7 @@
|
|||||||
|
|
||||||
const confirmVersionRestore = (row, version, label = null, onConfirm) => {
|
const confirmVersionRestore = (row, version, label = null, onConfirm) => {
|
||||||
if (!version) return;
|
if (!version) return;
|
||||||
const timestamp = (version.archived_at || version.last_modified) ? new Date(version.archived_at || version.last_modified).toLocaleString() : version.version_id;
|
const timestamp = version.archived_at ? new Date(version.archived_at).toLocaleString() : version.version_id;
|
||||||
const sizeLabel = formatBytes(Number(version.size) || 0);
|
const sizeLabel = formatBytes(Number(version.size) || 0);
|
||||||
const reasonLabel = describeVersionReason(version.reason);
|
const reasonLabel = describeVersionReason(version.reason);
|
||||||
const targetLabel = label || row?.dataset.key || 'this object';
|
const targetLabel = label || row?.dataset.key || 'this object';
|
||||||
@@ -1651,7 +1610,7 @@
|
|||||||
|
|
||||||
const latestCell = document.createElement('td');
|
const latestCell = document.createElement('td');
|
||||||
if (item.latest) {
|
if (item.latest) {
|
||||||
const ts = (item.latest.archived_at || item.latest.last_modified) ? new Date(item.latest.archived_at || item.latest.last_modified).toLocaleString() : item.latest.version_id;
|
const ts = item.latest.archived_at ? new Date(item.latest.archived_at).toLocaleString() : item.latest.version_id;
|
||||||
const sizeLabel = formatBytes(Number(item.latest.size) || 0);
|
const sizeLabel = formatBytes(Number(item.latest.size) || 0);
|
||||||
latestCell.innerHTML = `<div class="small">${ts}</div><div class="text-muted small">${sizeLabel} · ${describeVersionReason(item.latest.reason)}</div>`;
|
latestCell.innerHTML = `<div class="small">${ts}</div><div class="text-muted small">${sizeLabel} · ${describeVersionReason(item.latest.reason)}</div>`;
|
||||||
} else {
|
} else {
|
||||||
@@ -1778,15 +1737,6 @@
|
|||||||
loadArchivedObjects();
|
loadArchivedObjects();
|
||||||
}
|
}
|
||||||
|
|
||||||
const propertiesTab = document.getElementById('properties-tab');
|
|
||||||
if (propertiesTab) {
|
|
||||||
propertiesTab.addEventListener('shown.bs.tab', () => {
|
|
||||||
if (archivedCard && archivedEndpoint) {
|
|
||||||
loadArchivedObjects();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function restoreVersion(row, version) {
|
async function restoreVersion(row, version) {
|
||||||
if (!row || !version?.version_id) return;
|
if (!row || !version?.version_id) return;
|
||||||
const template = row.dataset.restoreTemplate;
|
const template = row.dataset.restoreTemplate;
|
||||||
@@ -1835,7 +1785,7 @@
|
|||||||
badge.textContent = `#${versionNumber}`;
|
badge.textContent = `#${versionNumber}`;
|
||||||
const title = document.createElement('div');
|
const title = document.createElement('div');
|
||||||
title.className = 'fw-semibold small';
|
title.className = 'fw-semibold small';
|
||||||
const timestamp = (entry.archived_at || entry.last_modified) ? new Date(entry.archived_at || entry.last_modified).toLocaleString() : entry.version_id;
|
const timestamp = entry.archived_at ? new Date(entry.archived_at).toLocaleString() : entry.version_id;
|
||||||
title.textContent = timestamp;
|
title.textContent = timestamp;
|
||||||
heading.appendChild(badge);
|
heading.appendChild(badge);
|
||||||
heading.appendChild(title);
|
heading.appendChild(title);
|
||||||
@@ -2866,16 +2816,7 @@
|
|||||||
uploadFileInput.value = '';
|
uploadFileInput.value = '';
|
||||||
}
|
}
|
||||||
|
|
||||||
const previousKey = activeRow?.dataset.key || null;
|
loadObjects(false);
|
||||||
loadObjects(false).then(() => {
|
|
||||||
if (previousKey) {
|
|
||||||
const newRow = document.querySelector(`[data-object-row][data-key="${CSS.escape(previousKey)}"]`);
|
|
||||||
if (newRow) {
|
|
||||||
selectRow(newRow);
|
|
||||||
if (versioningEnabled) loadObjectVersions(newRow, { force: true });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
const successCount = uploadSuccessFiles.length;
|
const successCount = uploadSuccessFiles.length;
|
||||||
const errorCount = uploadErrorFiles.length;
|
const errorCount = uploadErrorFiles.length;
|
||||||
@@ -4213,47 +4154,6 @@
|
|||||||
var archivedCardEl = document.getElementById('archived-objects-card');
|
var archivedCardEl = document.getElementById('archived-objects-card');
|
||||||
if (archivedCardEl) {
|
if (archivedCardEl) {
|
||||||
archivedCardEl.style.display = enabled ? '' : 'none';
|
archivedCardEl.style.display = enabled ? '' : 'none';
|
||||||
} else if (enabled) {
|
|
||||||
var endpoint = window.BucketDetailConfig?.endpoints?.archivedObjects || '';
|
|
||||||
if (endpoint) {
|
|
||||||
var html = '<div class="card shadow-sm mt-4" id="archived-objects-card" data-archived-endpoint="' + endpoint + '">' +
|
|
||||||
'<div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">' +
|
|
||||||
'<div class="d-flex align-items-center">' +
|
|
||||||
'<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" class="text-warning me-2" viewBox="0 0 16 16">' +
|
|
||||||
'<path d="M0 2a1 1 0 0 1 1-1h14a1 1 0 0 1 1 1v2a1 1 0 0 1-1 1v7.5a2.5 2.5 0 0 1-2.5 2.5h-9A2.5 2.5 0 0 1 1 12.5V5a1 1 0 0 1-1-1V2zm2 3v7.5A1.5 1.5 0 0 0 3.5 14h9a1.5 1.5 0 0 0 1.5-1.5V5H2zm13-3H1v2h14V2zM5 7.5a.5.5 0 0 1 .5-.5h5a.5.5 0 0 1 0 1h-5a.5.5 0 0 1-.5-.5z"/>' +
|
|
||||||
'</svg><span class="fw-semibold">Archived Objects</span></div>' +
|
|
||||||
'<div class="d-flex align-items-center gap-2">' +
|
|
||||||
'<span class="badge text-bg-secondary" data-archived-count>0 items</span>' +
|
|
||||||
'<button class="btn btn-outline-secondary btn-sm" type="button" data-archived-refresh>' +
|
|
||||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
|
|
||||||
'<path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>' +
|
|
||||||
'<path d="M8 4.466V.534a.25.25 0 0 0-.41-.192L5.23 2.308a.25.25 0 0 0 0 .384l2.36 1.966A.25.25 0 0 0 8 4.466z"/>' +
|
|
||||||
'</svg>Refresh</button></div></div>' +
|
|
||||||
'<div class="card-body">' +
|
|
||||||
'<p class="text-muted small mb-3">Objects that have been deleted while versioning is enabled. Their previous versions remain available until you restore or purge them.</p>' +
|
|
||||||
'<div class="table-responsive"><table class="table table-sm table-hover align-middle mb-0">' +
|
|
||||||
'<thead class="table-light"><tr>' +
|
|
||||||
'<th scope="col"><svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1 text-muted" viewBox="0 0 16 16">' +
|
|
||||||
'<path d="M4 0h5.293A1 1 0 0 1 10 .293L13.707 4a1 1 0 0 1 .293.707V14a2 2 0 0 1-2 2H4a2 2 0 0 1-2-2V2a2 2 0 0 1 2-2zm5.5 1.5v2a1 1 0 0 0 1 1h2l-3-3z"/>' +
|
|
||||||
'</svg>Key</th>' +
|
|
||||||
'<th scope="col">Latest Version</th>' +
|
|
||||||
'<th scope="col" class="text-center">Versions</th>' +
|
|
||||||
'<th scope="col" class="text-end">Actions</th>' +
|
|
||||||
'</tr></thead>' +
|
|
||||||
'<tbody data-archived-body><tr><td colspan="4" class="text-center text-muted py-4">' +
|
|
||||||
'<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="mb-2 d-block mx-auto" viewBox="0 0 16 16">' +
|
|
||||||
'<path d="M0 2a1 1 0 0 1 1-1h14a1 1 0 0 1 1 1v2a1 1 0 0 1-1 1v7.5a2.5 2.5 0 0 1-2.5 2.5h-9A2.5 2.5 0 0 1 1 12.5V5a1 1 0 0 1-1-1V2zm2 3v7.5A1.5 1.5 0 0 0 3.5 14h9a1.5 1.5 0 0 0 1.5-1.5V5H2zm13-3H1v2h14V2zM5 7.5a.5.5 0 0 1 .5-.5h5a.5.5 0 0 1 0 1h-5a.5.5 0 0 1-.5-.5z"/>' +
|
|
||||||
'</svg>No archived objects</td></tr></tbody>' +
|
|
||||||
'</table></div></div></div>';
|
|
||||||
card.insertAdjacentHTML('afterend', html);
|
|
||||||
archivedCard = document.getElementById('archived-objects-card');
|
|
||||||
archivedBody = archivedCard.querySelector('[data-archived-body]');
|
|
||||||
archivedCountBadge = archivedCard.querySelector('[data-archived-count]');
|
|
||||||
archivedRefreshButton = archivedCard.querySelector('[data-archived-refresh]');
|
|
||||||
archivedEndpoint = endpoint;
|
|
||||||
archivedRefreshButton.addEventListener('click', function() { loadArchivedObjects(); });
|
|
||||||
loadArchivedObjects();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var dropZone = document.getElementById('objects-drop-zone');
|
var dropZone = document.getElementById('objects-drop-zone');
|
||||||
@@ -4261,15 +4161,6 @@
|
|||||||
dropZone.setAttribute('data-versioning', enabled ? 'true' : 'false');
|
dropZone.setAttribute('data-versioning', enabled ? 'true' : 'false');
|
||||||
}
|
}
|
||||||
|
|
||||||
var bulkPurgeWrap = document.getElementById('bulkDeletePurgeWrap');
|
|
||||||
if (bulkPurgeWrap) {
|
|
||||||
bulkPurgeWrap.classList.toggle('d-none', !enabled);
|
|
||||||
}
|
|
||||||
var singleDeleteVerWrap = document.getElementById('deleteObjectVersioningWrap');
|
|
||||||
if (singleDeleteVerWrap) {
|
|
||||||
singleDeleteVerWrap.classList.toggle('d-none', !enabled);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!enabled) {
|
if (!enabled) {
|
||||||
var newForm = document.getElementById('enableVersioningForm');
|
var newForm = document.getElementById('enableVersioningForm');
|
||||||
if (newForm) {
|
if (newForm) {
|
||||||
|
|||||||
@@ -171,7 +171,6 @@
|
|||||||
data-bulk-download-endpoint="{{ url_for('ui.bulk_download_objects', bucket_name=bucket_name) }}"
|
data-bulk-download-endpoint="{{ url_for('ui.bulk_download_objects', bucket_name=bucket_name) }}"
|
||||||
data-folders-url="{{ folders_url }}"
|
data-folders-url="{{ folders_url }}"
|
||||||
data-buckets-for-copy-url="{{ buckets_for_copy_url }}"
|
data-buckets-for-copy-url="{{ buckets_for_copy_url }}"
|
||||||
data-bucket-total-objects="{{ bucket_stats.get('objects', 0) }}"
|
|
||||||
>
|
>
|
||||||
<table class="table table-hover align-middle mb-0" id="objects-table" style="table-layout: fixed;">
|
<table class="table table-hover align-middle mb-0" id="objects-table" style="table-layout: fixed;">
|
||||||
<thead class="table-light">
|
<thead class="table-light">
|
||||||
@@ -2273,11 +2272,13 @@
|
|||||||
</div>
|
</div>
|
||||||
<ul class="list-group mb-3" id="bulkDeleteList" style="max-height: 200px; overflow-y: auto;"></ul>
|
<ul class="list-group mb-3" id="bulkDeleteList" style="max-height: 200px; overflow-y: auto;"></ul>
|
||||||
<div class="text-muted small" id="bulkDeleteStatus"></div>
|
<div class="text-muted small" id="bulkDeleteStatus"></div>
|
||||||
<div class="form-check mt-3 p-3 bg-body-tertiary rounded-3 {% if not versioning_enabled %}d-none{% endif %}" id="bulkDeletePurgeWrap">
|
{% if versioning_enabled %}
|
||||||
|
<div class="form-check mt-3 p-3 bg-body-tertiary rounded-3">
|
||||||
<input class="form-check-input" type="checkbox" id="bulkDeletePurge" />
|
<input class="form-check-input" type="checkbox" id="bulkDeletePurge" />
|
||||||
<label class="form-check-label" for="bulkDeletePurge">Also delete archived versions</label>
|
<label class="form-check-label" for="bulkDeletePurge">Also delete archived versions</label>
|
||||||
<div class="form-text">Removes any archived versions stored in the archive.</div>
|
<div class="form-text">Removes any archived versions stored in the archive.</div>
|
||||||
</div>
|
</div>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="modal-footer">
|
<div class="modal-footer">
|
||||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||||
@@ -2315,7 +2316,7 @@
|
|||||||
<div class="p-3 bg-body-tertiary rounded-3 mb-3">
|
<div class="p-3 bg-body-tertiary rounded-3 mb-3">
|
||||||
<code id="deleteObjectKey" class="d-block text-break"></code>
|
<code id="deleteObjectKey" class="d-block text-break"></code>
|
||||||
</div>
|
</div>
|
||||||
<div id="deleteObjectVersioningWrap" class="{% if not versioning_enabled %}d-none{% endif %}">
|
{% if versioning_enabled %}
|
||||||
<div class="alert alert-warning d-flex align-items-start small mb-3" role="alert">
|
<div class="alert alert-warning d-flex align-items-start small mb-3" role="alert">
|
||||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="flex-shrink-0 me-2 mt-0" viewBox="0 0 16 16">
|
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="flex-shrink-0 me-2 mt-0" viewBox="0 0 16 16">
|
||||||
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
|
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
|
||||||
@@ -2327,7 +2328,7 @@
|
|||||||
<label class="form-check-label" for="deletePurgeVersions">Also delete all archived versions</label>
|
<label class="form-check-label" for="deletePurgeVersions">Also delete all archived versions</label>
|
||||||
<div class="form-text mb-0">Removes the live object and every stored version.</div>
|
<div class="form-text mb-0">Removes the live object and every stored version.</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="modal-footer">
|
<div class="modal-footer">
|
||||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||||
@@ -2770,8 +2771,7 @@
|
|||||||
window.BucketDetailConfig = {
|
window.BucketDetailConfig = {
|
||||||
endpoints: {
|
endpoints: {
|
||||||
versioning: "{{ url_for('ui.update_bucket_versioning', bucket_name=bucket_name) }}",
|
versioning: "{{ url_for('ui.update_bucket_versioning', bucket_name=bucket_name) }}",
|
||||||
bucketsOverview: "{{ url_for('ui.buckets_overview') }}",
|
bucketsOverview: "{{ url_for('ui.buckets_overview') }}"
|
||||||
archivedObjects: "{{ url_for('ui.archived_objects', bucket_name=bucket_name) }}"
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user