From b3dce8d13e0a0081d5e35287abe03fa7c8030135 Mon Sep 17 00:00:00 2001 From: kqjy Date: Thu, 1 Jan 2026 21:51:01 +0800 Subject: [PATCH] Fix Remove fallback ETag, make etag optional, fix multipart ETag storage, fix request entity too large error due to mishandled multipart uploads --- app/s3_api.py | 26 ++-- app/storage.py | 22 ++- app/ui.py | 12 +- static/css/main.css | 150 +++++++++++++++++-- templates/bucket_detail.html | 275 +++++++++++++++++++++++++++++++---- 5 files changed, 417 insertions(+), 68 deletions(-) diff --git a/app/s3_api.py b/app/s3_api.py index 25a3dd1..e87cf15 100644 --- a/app/s3_api.py +++ b/app/s3_api.py @@ -1314,7 +1314,8 @@ def _bucket_list_versions_handler(bucket_name: str) -> Response: SubElement(version, "VersionId").text = "null" SubElement(version, "IsLatest").text = "true" SubElement(version, "LastModified").text = obj.last_modified.strftime("%Y-%m-%dT%H:%M:%S.000Z") - SubElement(version, "ETag").text = f'"{obj.etag}"' + if obj.etag: + SubElement(version, "ETag").text = f'"{obj.etag}"' SubElement(version, "Size").text = str(obj.size) SubElement(version, "StorageClass").text = "STANDARD" @@ -2178,10 +2179,11 @@ def bucket_handler(bucket_name: str) -> Response: obj_el = SubElement(root, "Contents") SubElement(obj_el, "Key").text = meta.key SubElement(obj_el, "LastModified").text = meta.last_modified.isoformat() - SubElement(obj_el, "ETag").text = f'"{meta.etag}"' + if meta.etag: + SubElement(obj_el, "ETag").text = f'"{meta.etag}"' SubElement(obj_el, "Size").text = str(meta.size) SubElement(obj_el, "StorageClass").text = "STANDARD" - + for cp in common_prefixes: cp_el = SubElement(root, "CommonPrefixes") SubElement(cp_el, "Prefix").text = cp @@ -2194,15 +2196,16 @@ def bucket_handler(bucket_name: str) -> Response: SubElement(root, "IsTruncated").text = "true" if is_truncated else "false" if delimiter: SubElement(root, "Delimiter").text = delimiter - + if is_truncated and delimiter and next_marker: SubElement(root, "NextMarker").text = next_marker - + for meta in objects: obj_el = SubElement(root, "Contents") SubElement(obj_el, "Key").text = meta.key SubElement(obj_el, "LastModified").text = meta.last_modified.isoformat() - SubElement(obj_el, "ETag").text = f'"{meta.etag}"' + if meta.etag: + SubElement(obj_el, "ETag").text = f'"{meta.etag}"' SubElement(obj_el, "Size").text = str(meta.size) for cp in common_prefixes: @@ -2282,7 +2285,8 @@ def object_handler(bucket_name: str, object_key: str): extra={"bucket": bucket_name, "key": object_key, "size": meta.size}, ) response = Response(status=200) - response.headers["ETag"] = f'"{meta.etag}"' + if meta.etag: + response.headers["ETag"] = f'"{meta.etag}"' _notifications().emit_object_created( bucket_name, @@ -2725,7 +2729,8 @@ def _copy_object(dest_bucket: str, dest_key: str, copy_source: str) -> Response: root = Element("CopyObjectResult") SubElement(root, "LastModified").text = meta.last_modified.isoformat() - SubElement(root, "ETag").text = f'"{meta.etag}"' + if meta.etag: + SubElement(root, "ETag").text = f'"{meta.etag}"' return _xml_response(root) @@ -2947,8 +2952,9 @@ def _complete_multipart_upload(bucket_name: str, object_key: str) -> Response: SubElement(root, "Location").text = location SubElement(root, "Bucket").text = bucket_name SubElement(root, "Key").text = object_key - SubElement(root, "ETag").text = f'"{meta.etag}"' - + if meta.etag: + SubElement(root, "ETag").text = f'"{meta.etag}"' + return _xml_response(root) diff --git a/app/storage.py b/app/storage.py index 013d8d6..14d522b 100644 --- a/app/storage.py +++ b/app/storage.py @@ -90,7 +90,7 @@ class ObjectMeta: key: str size: int last_modified: datetime - etag: str + etag: Optional[str] = None metadata: Optional[Dict[str, str]] = None @@ -1079,11 +1079,6 @@ class ObjectStorage: checksum.update(data) target.write(data) - metadata = manifest.get("metadata") - if metadata: - self._write_metadata(bucket_id, safe_key, metadata) - else: - self._delete_metadata(bucket_id, safe_key) except BlockingIOError: raise StorageError("Another upload to this key is in progress") finally: @@ -1097,12 +1092,18 @@ class ObjectStorage: self._invalidate_bucket_stats_cache(bucket_id) stat = destination.stat() - # Performance: Lazy update - only update the affected key instead of invalidating whole cache + etag = checksum.hexdigest() + metadata = manifest.get("metadata") + + internal_meta = {"__etag__": etag, "__size__": str(stat.st_size)} + combined_meta = {**internal_meta, **(metadata or {})} + self._write_metadata(bucket_id, safe_key, combined_meta) + obj_meta = ObjectMeta( key=safe_key.as_posix(), size=stat.st_size, last_modified=datetime.fromtimestamp(stat.st_mtime, timezone.utc), - etag=checksum.hexdigest(), + etag=etag, metadata=metadata, ) self._update_object_cache_entry(bucket_id, safe_key.as_posix(), obj_meta) @@ -1369,10 +1370,7 @@ class ObjectStorage: stat = entry.stat() etag = meta_cache.get(key) - - if not etag: - etag = f'"{stat.st_size}-{int(stat.st_mtime)}"' - + objects[key] = ObjectMeta( key=key, size=stat.st_size, diff --git a/app/ui.py b/app/ui.py index c48a26e..34cbb4b 100644 --- a/app/ui.py +++ b/app/ui.py @@ -563,6 +563,7 @@ def initiate_multipart_upload(bucket_name: str): @ui_bp.put("/buckets//multipart//parts") +@limiter.exempt def upload_multipart_part(bucket_name: str, upload_id: str): principal = _current_principal() try: @@ -606,9 +607,14 @@ def complete_multipart_upload(bucket_name: str, upload_id: str): normalized.append({"part_number": number, "etag": etag}) try: result = _storage().complete_multipart_upload(bucket_name, upload_id, normalized) - _replication().trigger_replication(bucket_name, result["key"]) - - return jsonify(result) + _replication().trigger_replication(bucket_name, result.key) + + return jsonify({ + "key": result.key, + "size": result.size, + "etag": result.etag, + "last_modified": result.last_modified.isoformat() if result.last_modified else None, + }) except StorageError as exc: return jsonify({"error": str(exc)}), 400 diff --git a/static/css/main.css b/static/css/main.css index 4c9749b..1d39238 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -515,22 +515,77 @@ code { display: flex; flex-direction: column; gap: 0.75rem; + max-height: 300px; + overflow-y: auto; } .upload-progress-item { border: 1px solid var(--myfsio-card-border); - border-radius: 0.5rem; + border-radius: 0.75rem; background-color: var(--myfsio-card-bg); - padding: 0.75rem 0.9rem; - transition: border-color 0.2s ease, background-color 0.2s ease; + padding: 0.875rem 1rem; + transition: border-color 0.2s ease, background-color 0.2s ease, box-shadow 0.2s ease; +} + +.upload-progress-item[data-state='uploading'] { + border-color: rgba(59, 130, 246, 0.4); + box-shadow: 0 0 0 3px rgba(59, 130, 246, 0.08); } .upload-progress-item[data-state='success'] { border-color: rgba(34, 197, 94, 0.6); + background-color: rgba(34, 197, 94, 0.04); } .upload-progress-item[data-state='error'] { border-color: rgba(239, 68, 68, 0.7); + background-color: rgba(239, 68, 68, 0.04); +} + +.upload-progress-item .file-name { + font-weight: 500; + word-break: break-all; + margin-bottom: 0.25rem; +} + +.upload-progress-item .file-size { + font-size: 0.75rem; + color: var(--myfsio-muted); +} + +.upload-progress-item .upload-status { + font-size: 0.8rem; + color: var(--myfsio-muted); +} + +.upload-progress-item .upload-status.success { + color: #16a34a; +} + +.upload-progress-item .upload-status.error { + color: #dc2626; +} + +.upload-progress-item .progress-container { + margin-top: 0.5rem; +} + +.upload-progress-item .progress { + height: 6px; + border-radius: 999px; + overflow: hidden; +} + +.upload-progress-item .progress-bar { + transition: width 0.2s ease; +} + +.upload-progress-item .progress-text { + font-size: 0.7rem; + color: var(--myfsio-muted); + margin-top: 0.25rem; + display: flex; + justify-content: space-between; } .progress-thin { @@ -542,6 +597,14 @@ code { background-color: rgba(248, 250, 252, 0.15); } +[data-theme='dark'] .upload-progress-item .upload-status.success { + color: #4ade80; +} + +[data-theme='dark'] .upload-progress-item .upload-status.error { + color: #f87171; +} + #deleteObjectKey { word-break: break-all; max-width: 100%; @@ -583,6 +646,20 @@ code { border-color: #3b82f6; } +.upload-dropzone.upload-locked { + opacity: 0.5; + cursor: not-allowed; + background-color: var(--myfsio-preview-bg); +} + +.upload-dropzone.upload-locked::after { + content: 'Upload in progress...'; + display: block; + margin-top: 0.5rem; + font-size: 0.8rem; + color: var(--myfsio-muted); +} + .metadata-stack .metadata-entry + .metadata-entry { margin-top: 0.75rem; } @@ -620,13 +697,14 @@ code { } .objects-table-container thead th { - background-color: #f8f9fa; - border-bottom: 1px solid var(--myfsio-card-border); - box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); + background-color: var(--myfsio-preview-bg); + border-bottom: 2px solid var(--myfsio-card-border); + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.04); } [data-theme='dark'] .objects-table-container thead th { - background-color: #1e293b; + background-color: #1a2234; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.15); } .btn-group form { display: inline; } @@ -636,36 +714,77 @@ code { .table { color: var(--myfsio-text); background-color: var(--myfsio-card-bg); + border-collapse: separate; + border-spacing: 0; + font-size: 0.9rem; } .table th, .table td { border-color: var(--myfsio-card-border); + padding: 0.875rem 1rem; + vertical-align: middle; +} + +.table th { + font-weight: 600; + font-size: 0.75rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--myfsio-muted); + border-bottom: 2px solid var(--myfsio-card-border); +} + +.table td { + border-bottom: 1px solid rgba(0, 0, 0, 0.05); +} + +[data-theme='dark'] .table td { + border-bottom: 1px solid rgba(255, 255, 255, 0.05); +} + +.table tbody tr:last-child td { + border-bottom: none; } .table-light th { - background-color: rgba(15, 23, 42, 0.04); + background-color: var(--myfsio-preview-bg); } [data-theme='dark'] .table-light th { - background-color: rgba(248, 250, 252, 0.05); - color: var(--myfsio-text); + background-color: rgba(248, 250, 252, 0.03); + color: var(--myfsio-muted); +} + +.table-hover tbody tr { + transition: all 0.15s ease; } .table-hover tbody tr:hover { background-color: var(--myfsio-hover-bg); cursor: pointer; - transition: background-color 0.15s ease; +} + +.table-hover tbody tr:hover td { + border-bottom-color: transparent; } .table thead { - background-color: rgba(15, 23, 42, 0.04); - color: var(--myfsio-text); + background-color: var(--myfsio-preview-bg); + color: var(--myfsio-muted); } [data-theme='dark'] .table thead { - background-color: rgba(248, 250, 252, 0.05); - color: var(--myfsio-text); + background-color: rgba(248, 250, 252, 0.03); + color: var(--myfsio-muted); +} + +.table-striped tbody tr:nth-of-type(odd) { + background-color: rgba(0, 0, 0, 0.015); +} + +[data-theme='dark'] .table-striped tbody tr:nth-of-type(odd) { + background-color: rgba(255, 255, 255, 0.02); } .form-control, @@ -1776,6 +1895,7 @@ body.theme-transitioning * { gap: 0.5rem; align-items: center; flex: 1; + justify-content: flex-end; } @media (max-width: 640px) { diff --git a/templates/bucket_detail.html b/templates/bucket_detail.html index 50c4b4c..270e10f 100644 --- a/templates/bucket_detail.html +++ b/templates/bucket_detail.html @@ -451,6 +451,12 @@
+
+ + + + This preset is read-only. Select Custom JSON to edit the policy manually. +
@@ -3320,8 +3326,14 @@ } }; + const policyReadonlyHint = document.getElementById('policyReadonlyHint'); + const applyPolicyPreset = (preset) => { if (!policyTextarea || !policyMode) return; + const isPresetMode = preset === 'private' || preset === 'public'; + if (policyReadonlyHint) { + policyReadonlyHint.classList.toggle('d-none', !isPresetMode); + } switch (preset) { case 'private': setPolicyTextareaState(true); @@ -3329,7 +3341,7 @@ policyMode.value = 'delete'; break; case 'public': - setPolicyTextareaState(false); + setPolicyTextareaState(true); policyTextarea.value = publicPolicyTemplate || ''; policyMode.value = 'upsert'; break; @@ -4160,42 +4172,245 @@ if (bulkUploadErrorList) bulkUploadErrorList.innerHTML = ''; if (uploadSubmitBtn) uploadSubmitBtn.disabled = false; if (uploadFileInput) uploadFileInput.disabled = false; + const progressStack = document.querySelector('[data-upload-progress]'); + if (progressStack) progressStack.innerHTML = ''; + if (uploadDropZone) { + uploadDropZone.classList.remove('upload-locked'); + uploadDropZone.style.pointerEvents = ''; + } isUploading = false; }; - const uploadSingleFile = async (file, keyPrefix = '', metadata = null) => { - const formData = new FormData(); - formData.append('object', file); + const MULTIPART_THRESHOLD = 8 * 1024 * 1024; + const CHUNK_SIZE = 8 * 1024 * 1024; + const uploadProgressStack = document.querySelector('[data-upload-progress]'); + const multipartInitUrl = uploadForm.dataset.multipartInitUrl; + const multipartPartTemplate = uploadForm.dataset.multipartPartTemplate; + const multipartCompleteTemplate = uploadForm.dataset.multipartCompleteTemplate; + const multipartAbortTemplate = uploadForm.dataset.multipartAbortTemplate; + + const createProgressItem = (file) => { + const item = document.createElement('div'); + item.className = 'upload-progress-item'; + item.dataset.state = 'uploading'; + item.innerHTML = ` +
+
+
${escapeHtml(file.name)}
+
${formatBytes(file.size)}
+
+
Preparing...
+
+
+
+
+
+
+ 0 B + 0% +
+
+ `; + return item; + }; + + const updateProgressItem = (item, { loaded, total, status, state, error }) => { + if (state) item.dataset.state = state; + const statusEl = item.querySelector('.upload-status'); + const progressBar = item.querySelector('.progress-bar'); + const progressLoaded = item.querySelector('.progress-loaded'); + const progressPercent = item.querySelector('.progress-percent'); + + if (status) { + statusEl.textContent = status; + statusEl.className = 'upload-status text-end ms-2'; + if (state === 'success') statusEl.classList.add('success'); + if (state === 'error') statusEl.classList.add('error'); + } + if (typeof loaded === 'number' && typeof total === 'number' && total > 0) { + const percent = Math.round((loaded / total) * 100); + progressBar.style.width = `${percent}%`; + progressLoaded.textContent = `${formatBytes(loaded)} / ${formatBytes(total)}`; + progressPercent.textContent = `${percent}%`; + } + if (error) { + const progressContainer = item.querySelector('.progress-container'); + if (progressContainer) { + progressContainer.innerHTML = `
${escapeHtml(error)}
`; + } + } + }; + + const uploadMultipart = async (file, objectKey, metadata, progressItem) => { + const csrfToken = document.querySelector('input[name="csrf_token"]')?.value; + + updateProgressItem(progressItem, { status: 'Initiating...', loaded: 0, total: file.size }); + const initResp = await fetch(multipartInitUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-CSRFToken': csrfToken || '' }, + body: JSON.stringify({ object_key: objectKey, metadata }) + }); + if (!initResp.ok) { + const err = await initResp.json().catch(() => ({})); + throw new Error(err.error || 'Failed to initiate upload'); + } + const { upload_id } = await initResp.json(); + + const partUrl = multipartPartTemplate.replace('UPLOAD_ID_PLACEHOLDER', upload_id); + const completeUrl = multipartCompleteTemplate.replace('UPLOAD_ID_PLACEHOLDER', upload_id); + const abortUrl = multipartAbortTemplate.replace('UPLOAD_ID_PLACEHOLDER', upload_id); + + const parts = []; + const totalParts = Math.ceil(file.size / CHUNK_SIZE); + let uploadedBytes = 0; + + try { + for (let partNumber = 1; partNumber <= totalParts; partNumber++) { + const start = (partNumber - 1) * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, file.size); + const chunk = file.slice(start, end); + + updateProgressItem(progressItem, { + status: `Part ${partNumber}/${totalParts}`, + loaded: uploadedBytes, + total: file.size + }); + + const partResp = await fetch(`${partUrl}?partNumber=${partNumber}`, { + method: 'PUT', + headers: { 'X-CSRFToken': csrfToken || '' }, + body: chunk + }); + + if (!partResp.ok) { + const err = await partResp.json().catch(() => ({})); + throw new Error(err.error || `Part ${partNumber} failed`); + } + + const partData = await partResp.json(); + parts.push({ part_number: partNumber, etag: partData.etag }); + uploadedBytes += chunk.size; + + updateProgressItem(progressItem, { + loaded: uploadedBytes, + total: file.size + }); + } + + updateProgressItem(progressItem, { status: 'Completing...', loaded: file.size, total: file.size }); + const completeResp = await fetch(completeUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-CSRFToken': csrfToken || '' }, + body: JSON.stringify({ parts }) + }); + + if (!completeResp.ok) { + const err = await completeResp.json().catch(() => ({})); + throw new Error(err.error || 'Failed to complete upload'); + } + + return await completeResp.json(); + } catch (err) { + try { + await fetch(abortUrl, { method: 'DELETE', headers: { 'X-CSRFToken': csrfToken || '' } }); + } catch {} + throw err; + } + }; + + const uploadRegular = async (file, objectKey, metadata, progressItem) => { + return new Promise((resolve, reject) => { + const formData = new FormData(); + formData.append('object', file); + formData.append('object_key', objectKey); + if (metadata) formData.append('metadata', JSON.stringify(metadata)); + const csrfToken = document.querySelector('input[name="csrf_token"]')?.value; + if (csrfToken) formData.append('csrf_token', csrfToken); + + const xhr = new XMLHttpRequest(); + xhr.open('POST', uploadForm.action, true); + xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest'); + + xhr.upload.addEventListener('progress', (e) => { + if (e.lengthComputable) { + updateProgressItem(progressItem, { + status: 'Uploading...', + loaded: e.loaded, + total: e.total + }); + } + }); + + xhr.addEventListener('load', () => { + if (xhr.status >= 200 && xhr.status < 300) { + try { + const data = JSON.parse(xhr.responseText); + if (data.status === 'error') { + reject(new Error(data.message || 'Upload failed')); + } else { + resolve(data); + } + } catch { + resolve({}); + } + } else { + try { + const data = JSON.parse(xhr.responseText); + reject(new Error(data.message || `Upload failed (${xhr.status})`)); + } catch { + reject(new Error(`Upload failed (${xhr.status})`)); + } + } + }); + + xhr.addEventListener('error', () => reject(new Error('Network error'))); + xhr.addEventListener('abort', () => reject(new Error('Upload aborted'))); + + xhr.send(formData); + }); + }; + + const uploadSingleFile = async (file, keyPrefix = '', metadata = null, progressItem = null) => { const objectKey = keyPrefix ? `${keyPrefix}${file.name}` : file.name; - formData.append('object_key', objectKey); - if (metadata) { - formData.append('metadata', JSON.stringify(metadata)); + const shouldUseMultipart = file.size >= MULTIPART_THRESHOLD && multipartInitUrl; + + if (!progressItem && uploadProgressStack) { + progressItem = createProgressItem(file); + uploadProgressStack.appendChild(progressItem); } - const csrfToken = document.querySelector('input[name="csrf_token"]')?.value; - if (csrfToken) { - formData.append('csrf_token', csrfToken); - } - - const response = await fetch(uploadForm.action, { - method: 'POST', - body: formData, - headers: { - 'X-Requested-With': 'XMLHttpRequest' + try { + let result; + if (shouldUseMultipart) { + updateProgressItem(progressItem, { status: 'Multipart upload...', loaded: 0, total: file.size }); + result = await uploadMultipart(file, objectKey, metadata, progressItem); + } else { + updateProgressItem(progressItem, { status: 'Uploading...', loaded: 0, total: file.size }); + result = await uploadRegular(file, objectKey, metadata, progressItem); } - }); - - const data = await response.json().catch(() => ({})); - if (!response.ok || data.status === 'error') { - throw new Error(data.message || 'Upload failed'); + updateProgressItem(progressItem, { state: 'success', status: 'Complete', loaded: file.size, total: file.size }); + return result; + } catch (err) { + updateProgressItem(progressItem, { state: 'error', status: 'Failed', error: err.message }); + throw err; + } + }; + + const setUploadLockState = (locked) => { + if (uploadDropZone) { + uploadDropZone.classList.toggle('upload-locked', locked); + uploadDropZone.style.pointerEvents = locked ? 'none' : ''; + } + if (uploadFileInput) { + uploadFileInput.disabled = locked; } - return data; }; const performBulkUpload = async (files) => { if (isUploading || !files || files.length === 0) return; - + isUploading = true; + setUploadLockState(true); const keyPrefix = (uploadKeyPrefix?.value || '').trim(); const metadataRaw = uploadForm.querySelector('textarea[name="metadata"]')?.value?.trim(); let metadata = null; @@ -4256,14 +4471,12 @@ } isUploading = false; + setUploadLockState(false); if (successFiles.length > 0) { - if (uploadBtnText) uploadBtnText.textContent = 'Refreshing...'; - window.setTimeout(() => window.location.reload(), 800); } else { - if (uploadSubmitBtn) uploadSubmitBtn.disabled = false; if (uploadFileInput) uploadFileInput.disabled = false; } @@ -4271,11 +4484,15 @@ refreshUploadDropLabel(); uploadFileInput.addEventListener('change', () => { + if (isUploading) return; refreshUploadDropLabel(); updateUploadBtnText(); resetUploadUI(); }); - uploadDropZone?.addEventListener('click', () => uploadFileInput?.click()); + uploadDropZone?.addEventListener('click', () => { + if (isUploading) return; + uploadFileInput?.click(); + }); uploadForm.addEventListener('submit', async (event) => { const files = uploadFileInput.files; @@ -4337,6 +4554,7 @@ ['dragenter', 'dragover'].forEach((eventName) => { target.addEventListener(eventName, (event) => { preventDefaults(event); + if (isUploading) return; if (highlightClass) { target.classList.add(highlightClass); } @@ -4351,6 +4569,7 @@ }); }); target.addEventListener('drop', (event) => { + if (isUploading) return; if (!event.dataTransfer?.files?.length || !uploadFileInput) { return; }