Fix pausing replication and resuming replication does not continue the replication for the remaining pending objects; Improve Documentation
This commit is contained in:
33
app/ui.py
33
app/ui.py
@@ -415,7 +415,7 @@ def list_bucket_objects(bucket_name: str):
|
||||
except IamError as exc:
|
||||
return jsonify({"error": str(exc)}), 403
|
||||
|
||||
max_keys = min(int(request.args.get("max_keys", 1000)), 10000)
|
||||
max_keys = min(int(request.args.get("max_keys", 1000)), 100000)
|
||||
continuation_token = request.args.get("continuation_token") or None
|
||||
prefix = request.args.get("prefix") or None
|
||||
|
||||
@@ -434,6 +434,14 @@ def list_bucket_objects(bucket_name: str):
|
||||
except StorageError:
|
||||
versioning_enabled = False
|
||||
|
||||
# Pre-compute URL templates once (not per-object) for performance
|
||||
# Frontend will construct actual URLs by replacing KEY_PLACEHOLDER
|
||||
preview_template = url_for("ui.object_preview", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
delete_template = url_for("ui.delete_object", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
presign_template = url_for("ui.object_presign", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
versions_template = url_for("ui.object_versions", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
restore_template = url_for("ui.restore_object_version", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER", version_id="VERSION_ID_PLACEHOLDER")
|
||||
|
||||
objects_data = []
|
||||
for obj in result.objects:
|
||||
objects_data.append({
|
||||
@@ -442,13 +450,6 @@ def list_bucket_objects(bucket_name: str):
|
||||
"last_modified": obj.last_modified.isoformat(),
|
||||
"last_modified_display": obj.last_modified.strftime("%b %d, %Y %H:%M"),
|
||||
"etag": obj.etag,
|
||||
"metadata": obj.metadata or {},
|
||||
"preview_url": url_for("ui.object_preview", bucket_name=bucket_name, object_key=obj.key),
|
||||
"download_url": url_for("ui.object_preview", bucket_name=bucket_name, object_key=obj.key) + "?download=1",
|
||||
"presign_endpoint": url_for("ui.object_presign", bucket_name=bucket_name, object_key=obj.key),
|
||||
"delete_endpoint": url_for("ui.delete_object", bucket_name=bucket_name, object_key=obj.key),
|
||||
"versions_endpoint": url_for("ui.object_versions", bucket_name=bucket_name, object_key=obj.key),
|
||||
"restore_template": url_for("ui.restore_object_version", bucket_name=bucket_name, object_key=obj.key, version_id="VERSION_ID_PLACEHOLDER"),
|
||||
})
|
||||
|
||||
return jsonify({
|
||||
@@ -457,6 +458,14 @@ def list_bucket_objects(bucket_name: str):
|
||||
"next_continuation_token": result.next_continuation_token,
|
||||
"total_count": result.total_count,
|
||||
"versioning_enabled": versioning_enabled,
|
||||
"url_templates": {
|
||||
"preview": preview_template,
|
||||
"download": preview_template + "?download=1",
|
||||
"presign": presign_template,
|
||||
"delete": delete_template,
|
||||
"versions": versions_template,
|
||||
"restore": restore_template,
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -1458,11 +1467,17 @@ def update_bucket_replication(bucket_name: str):
|
||||
else:
|
||||
flash("No replication configuration to pause", "warning")
|
||||
elif action == "resume":
|
||||
from .replication import REPLICATION_MODE_ALL
|
||||
rule = _replication().get_rule(bucket_name)
|
||||
if rule:
|
||||
rule.enabled = True
|
||||
_replication().set_rule(rule)
|
||||
flash("Replication resumed", "success")
|
||||
# When resuming, sync any pending objects that accumulated while paused
|
||||
if rule.mode == REPLICATION_MODE_ALL:
|
||||
_replication().replicate_existing_objects(bucket_name)
|
||||
flash("Replication resumed. Syncing pending objects in background.", "success")
|
||||
else:
|
||||
flash("Replication resumed", "success")
|
||||
else:
|
||||
flash("No replication configuration to resume", "warning")
|
||||
elif action == "create":
|
||||
|
||||
@@ -173,14 +173,16 @@
|
||||
</div>
|
||||
<div class="d-flex align-items-center gap-1">
|
||||
<span class="text-muted">Batch</span>
|
||||
<select id="page-size-select" class="form-select form-select-sm py-0" style="width: auto; font-size: 0.75rem;">
|
||||
<select id="page-size-select" class="form-select form-select-sm py-0" style="width: auto; font-size: 0.75rem;" title="Number of objects to load per batch">
|
||||
<option value="1000">1K</option>
|
||||
<option value="5000" selected>5K</option>
|
||||
<option value="10000">10K</option>
|
||||
<option value="25000">25K</option>
|
||||
<option value="50000">50K</option>
|
||||
<option value="75000">75K</option>
|
||||
<option value="100000">100K</option>
|
||||
</select>
|
||||
<span class="text-muted">objects</span>
|
||||
<span class="text-muted">per batch</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1144,13 +1146,18 @@
|
||||
</div>
|
||||
|
||||
{% elif replication_rule and not replication_rule.enabled %}
|
||||
<div class="alert alert-warning d-flex align-items-center mb-4" role="alert">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="flex-shrink-0 me-2" viewBox="0 0 16 16">
|
||||
<div class="alert alert-warning d-flex align-items-start mb-4" role="alert">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="flex-shrink-0 me-2 mt-1" viewBox="0 0 16 16">
|
||||
<path d="M5.5 3.5A1.5 1.5 0 0 1 7 5v6a1.5 1.5 0 0 1-3 0V5a1.5 1.5 0 0 1 1.5-1.5zm5 0A1.5 1.5 0 0 1 12 5v6a1.5 1.5 0 0 1-3 0V5a1.5 1.5 0 0 1 1.5-1.5z"/>
|
||||
</svg>
|
||||
<div>
|
||||
<strong>Replication Paused</strong> —
|
||||
Replication is configured but currently paused. New uploads will not be replicated until resumed.
|
||||
<strong>Replication Paused</strong>
|
||||
<p class="mb-1">Replication is configured but currently paused. New uploads will not be replicated until resumed.</p>
|
||||
{% if replication_rule.mode == 'all' %}
|
||||
<p class="mb-0 small text-dark"><strong>Tip:</strong> When you resume, any objects uploaded while paused will be automatically synced to the target.</p>
|
||||
{% else %}
|
||||
<p class="mb-0 small text-dark"><strong>Note:</strong> Objects uploaded while paused will not be synced (mode: new_only). Consider switching to "All Objects" mode if you need to sync missed uploads.</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1882,6 +1889,13 @@
|
||||
let pageSize = 5000; // Load large batches for virtual scrolling
|
||||
let currentPrefix = ''; // Current folder prefix for navigation
|
||||
let allObjects = []; // All loaded object metadata (lightweight)
|
||||
let urlTemplates = null; // URL templates from API for constructing object URLs
|
||||
|
||||
// Helper to build URL from template by replacing KEY_PLACEHOLDER with encoded key
|
||||
const buildUrlFromTemplate = (template, key) => {
|
||||
if (!template) return '';
|
||||
return template.replace('KEY_PLACEHOLDER', encodeURIComponent(key).replace(/%2F/g, '/'));
|
||||
};
|
||||
|
||||
// Virtual scrolling state
|
||||
const ROW_HEIGHT = 53; // Height of each table row in pixels
|
||||
@@ -2223,22 +2237,26 @@
|
||||
objectsLoadingRow.remove();
|
||||
}
|
||||
|
||||
// Store lightweight object metadata (no DOM elements!)
|
||||
if (data.url_templates && !urlTemplates) {
|
||||
urlTemplates = data.url_templates;
|
||||
}
|
||||
|
||||
data.objects.forEach(obj => {
|
||||
loadedObjectCount++;
|
||||
const key = obj.key;
|
||||
allObjects.push({
|
||||
key: obj.key,
|
||||
key: key,
|
||||
size: obj.size,
|
||||
lastModified: obj.last_modified,
|
||||
lastModifiedDisplay: obj.last_modified_display,
|
||||
etag: obj.etag,
|
||||
previewUrl: obj.preview_url,
|
||||
downloadUrl: obj.download_url,
|
||||
presignEndpoint: obj.presign_endpoint,
|
||||
deleteEndpoint: obj.delete_endpoint,
|
||||
metadata: JSON.stringify(obj.metadata || {}),
|
||||
versionsEndpoint: obj.versions_endpoint,
|
||||
restoreTemplate: obj.restore_template
|
||||
previewUrl: urlTemplates ? buildUrlFromTemplate(urlTemplates.preview, key) : '',
|
||||
downloadUrl: urlTemplates ? buildUrlFromTemplate(urlTemplates.download, key) : '',
|
||||
presignEndpoint: urlTemplates ? buildUrlFromTemplate(urlTemplates.presign, key) : '',
|
||||
deleteEndpoint: urlTemplates ? buildUrlFromTemplate(urlTemplates.delete, key) : '',
|
||||
metadata: '{}',
|
||||
versionsEndpoint: urlTemplates ? buildUrlFromTemplate(urlTemplates.versions, key) : '',
|
||||
restoreTemplate: urlTemplates ? urlTemplates.restore.replace('KEY_PLACEHOLDER', encodeURIComponent(key).replace(/%2F/g, '/')) : ''
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -407,10 +407,62 @@ curl -X POST {{ api_base }}/presign/demo/notes.txt \
|
||||
<span class="docs-section-kicker">07</span>
|
||||
<h2 class="h4 mb-0">API Examples</h2>
|
||||
</div>
|
||||
<p class="text-muted">Common operations using boto3.</p>
|
||||
<p class="text-muted">Common operations using popular SDKs and tools.</p>
|
||||
|
||||
<h5 class="mt-4">Multipart Upload</h5>
|
||||
<pre><code class="language-python">import boto3
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Python (boto3)</h3>
|
||||
<pre class="mb-4"><code class="language-python">import boto3
|
||||
|
||||
s3 = boto3.client(
|
||||
's3',
|
||||
endpoint_url='{{ api_base }}',
|
||||
aws_access_key_id='<access_key>',
|
||||
aws_secret_access_key='<secret_key>'
|
||||
)
|
||||
|
||||
# List buckets
|
||||
buckets = s3.list_buckets()['Buckets']
|
||||
|
||||
# Create bucket
|
||||
s3.create_bucket(Bucket='mybucket')
|
||||
|
||||
# Upload file
|
||||
s3.upload_file('local.txt', 'mybucket', 'remote.txt')
|
||||
|
||||
# Download file
|
||||
s3.download_file('mybucket', 'remote.txt', 'downloaded.txt')
|
||||
|
||||
# Generate presigned URL (valid 1 hour)
|
||||
url = s3.generate_presigned_url(
|
||||
'get_object',
|
||||
Params={'Bucket': 'mybucket', 'Key': 'remote.txt'},
|
||||
ExpiresIn=3600
|
||||
)</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">JavaScript (AWS SDK v3)</h3>
|
||||
<pre class="mb-4"><code class="language-javascript">import { S3Client, ListBucketsCommand, PutObjectCommand } from '@aws-sdk/client-s3';
|
||||
|
||||
const s3 = new S3Client({
|
||||
endpoint: '{{ api_base }}',
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: '<access_key>',
|
||||
secretAccessKey: '<secret_key>'
|
||||
},
|
||||
forcePathStyle: true // Required for S3-compatible services
|
||||
});
|
||||
|
||||
// List buckets
|
||||
const { Buckets } = await s3.send(new ListBucketsCommand({}));
|
||||
|
||||
// Upload object
|
||||
await s3.send(new PutObjectCommand({
|
||||
Bucket: 'mybucket',
|
||||
Key: 'hello.txt',
|
||||
Body: 'Hello, World!'
|
||||
}));</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Multipart Upload (Python)</h3>
|
||||
<pre class="mb-4"><code class="language-python">import boto3
|
||||
|
||||
s3 = boto3.client('s3', endpoint_url='{{ api_base }}')
|
||||
|
||||
@@ -418,9 +470,9 @@ s3 = boto3.client('s3', endpoint_url='{{ api_base }}')
|
||||
response = s3.create_multipart_upload(Bucket='mybucket', Key='large.bin')
|
||||
upload_id = response['UploadId']
|
||||
|
||||
# Upload parts
|
||||
# Upload parts (minimum 5MB each, except last part)
|
||||
parts = []
|
||||
chunks = [b'chunk1', b'chunk2'] # Example data chunks
|
||||
chunks = [b'chunk1...', b'chunk2...']
|
||||
for part_number, chunk in enumerate(chunks, start=1):
|
||||
response = s3.upload_part(
|
||||
Bucket='mybucket',
|
||||
@@ -438,6 +490,19 @@ s3.complete_multipart_upload(
|
||||
UploadId=upload_id,
|
||||
MultipartUpload={'Parts': parts}
|
||||
)</code></pre>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Presigned URLs for Sharing</h3>
|
||||
<pre class="mb-0"><code class="language-bash"># Generate a download link valid for 15 minutes
|
||||
curl -X POST "{{ api_base }}/presign/mybucket/photo.jpg" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-d '{"method": "GET", "expires_in": 900}'
|
||||
|
||||
# Generate an upload link (PUT) valid for 1 hour
|
||||
curl -X POST "{{ api_base }}/presign/mybucket/upload.bin" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-d '{"method": "PUT", "expires_in": 3600}'</code></pre>
|
||||
</div>
|
||||
</article>
|
||||
<article id="replication" class="card shadow-sm docs-section">
|
||||
@@ -487,6 +552,86 @@ s3.complete_multipart_upload(
|
||||
</p>
|
||||
</div>
|
||||
</article>
|
||||
<article id="versioning" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<span class="docs-section-kicker">09</span>
|
||||
<h2 class="h4 mb-0">Object Versioning</h2>
|
||||
</div>
|
||||
<p class="text-muted">Keep multiple versions of objects to protect against accidental deletions and overwrites. Restore previous versions at any time.</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Enabling Versioning</h3>
|
||||
<ol class="docs-steps mb-3">
|
||||
<li>Navigate to your bucket's <strong>Properties</strong> tab.</li>
|
||||
<li>Find the <strong>Versioning</strong> card and click <strong>Enable</strong>.</li>
|
||||
<li>All subsequent uploads will create new versions instead of overwriting.</li>
|
||||
</ol>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Version Operations</h3>
|
||||
<div class="table-responsive mb-3">
|
||||
<table class="table table-sm table-bordered small">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Operation</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><strong>View Versions</strong></td>
|
||||
<td>Click the version icon on any object to see all historical versions with timestamps and sizes.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Restore Version</strong></td>
|
||||
<td>Click <strong>Restore</strong> on any version to make it the current version (creates a copy).</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Delete Current</strong></td>
|
||||
<td>Deleting an object archives it. Previous versions remain accessible.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Purge All</strong></td>
|
||||
<td>Permanently delete an object and all its versions. This cannot be undone.</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">Archived Objects</h3>
|
||||
<p class="small text-muted mb-3">When you delete a versioned object, it becomes "archived" - the current version is removed but historical versions remain. The <strong>Archived</strong> tab shows these objects so you can restore them.</p>
|
||||
|
||||
<h3 class="h6 text-uppercase text-muted mt-4">API Usage</h3>
|
||||
<pre class="mb-3"><code class="language-bash"># Enable versioning
|
||||
curl -X PUT "{{ api_base }}/<bucket>?versioning" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||||
-d '{"Status": "Enabled"}'
|
||||
|
||||
# Get versioning status
|
||||
curl "{{ api_base }}/<bucket>?versioning" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# List object versions
|
||||
curl "{{ api_base }}/<bucket>?versions" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||||
|
||||
# Get specific version
|
||||
curl "{{ api_base }}/<bucket>/<key>?versionId=<version-id>" \
|
||||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||||
|
||||
<div class="alert alert-light border mb-0">
|
||||
<div class="d-flex gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1" viewBox="0 0 16 16">
|
||||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||||
</svg>
|
||||
<div>
|
||||
<strong>Storage Impact:</strong> Each version consumes storage. Enable quotas to limit total bucket size including all versions.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
<article id="quotas" class="card shadow-sm docs-section">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
@@ -709,6 +854,7 @@ curl -X DELETE "{{ api_base }}/kms/keys/{key-id}?waiting_period_days=30" \
|
||||
<li><a href="#api">REST endpoints</a></li>
|
||||
<li><a href="#examples">API Examples</a></li>
|
||||
<li><a href="#replication">Site Replication</a></li>
|
||||
<li><a href="#versioning">Object Versioning</a></li>
|
||||
<li><a href="#quotas">Bucket Quotas</a></li>
|
||||
<li><a href="#encryption">Encryption</a></li>
|
||||
<li><a href="#troubleshooting">Troubleshooting</a></li>
|
||||
|
||||
@@ -157,9 +157,14 @@ class TestPaginatedObjectListing:
|
||||
assert "last_modified" in obj
|
||||
assert "last_modified_display" in obj
|
||||
assert "etag" in obj
|
||||
assert "preview_url" in obj
|
||||
assert "download_url" in obj
|
||||
assert "delete_endpoint" in obj
|
||||
|
||||
# URLs are now returned as templates (not per-object) for performance
|
||||
assert "url_templates" in data
|
||||
templates = data["url_templates"]
|
||||
assert "preview" in templates
|
||||
assert "download" in templates
|
||||
assert "delete" in templates
|
||||
assert "KEY_PLACEHOLDER" in templates["preview"]
|
||||
|
||||
def test_bucket_detail_page_loads_without_objects(self, tmp_path):
|
||||
"""Bucket detail page should load even with many objects."""
|
||||
|
||||
Reference in New Issue
Block a user