MyFSIO v0.2.4 Release #16

Merged
kqjy merged 15 commits from next into main 2026-02-01 10:27:11 +00:00
4 changed files with 492 additions and 1 deletions
Showing only changes of commit 6b715851b9 - Show all commits

View File

@@ -318,6 +318,9 @@ class ReplicationManager:
def get_rule(self, bucket_name: str) -> Optional[ReplicationRule]:
return self._rules.get(bucket_name)
def list_rules(self) -> List[ReplicationRule]:
return list(self._rules.values())
def set_rule(self, rule: ReplicationRule) -> None:
old_rule = self._rules.get(rule.bucket_name)
was_all_mode = old_rule and old_rule.mode == REPLICATION_MODE_ALL if old_rule else False

195
app/ui.py
View File

@@ -2682,11 +2682,28 @@ def sites_dashboard():
peers = registry.list_peers()
connections = _connections().list()
replication = _replication()
all_rules = replication.list_rules()
peers_with_stats = []
for peer in peers:
buckets_syncing = 0
if peer.connection_id:
for rule in all_rules:
if rule.target_connection_id == peer.connection_id:
buckets_syncing += 1
peers_with_stats.append({
"peer": peer,
"buckets_syncing": buckets_syncing,
"has_connection": bool(peer.connection_id),
})
return render_template(
"sites.html",
principal=principal,
local_site=local_site,
peers=peers,
peers_with_stats=peers_with_stats,
connections=connections,
config_site_id=current_app.config.get("SITE_ID"),
config_site_endpoint=current_app.config.get("SITE_ENDPOINT"),
@@ -2784,6 +2801,9 @@ def add_peer_site():
registry.add_peer(peer)
flash(f"Peer site '{site_id}' added", "success")
if connection_id:
return redirect(url_for("ui.replication_wizard", site_id=site_id))
return redirect(url_for("ui.sites_dashboard"))
@@ -2891,6 +2911,181 @@ def check_peer_site_health(site_id: str):
return jsonify(result)
@ui_bp.get("/sites/peers/<site_id>/replication-wizard")
def replication_wizard(site_id: str):
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:*")
except IamError:
flash("Access denied", "danger")
return redirect(url_for("ui.sites_dashboard"))
registry = _site_registry()
peer = registry.get_peer(site_id)
if not peer:
flash(f"Peer site '{site_id}' not found", "danger")
return redirect(url_for("ui.sites_dashboard"))
if not peer.connection_id:
flash("This peer has no connection configured. Add a connection first to set up replication.", "warning")
return redirect(url_for("ui.sites_dashboard"))
connection = _connections().get(peer.connection_id)
if not connection:
flash(f"Connection '{peer.connection_id}' not found", "danger")
return redirect(url_for("ui.sites_dashboard"))
buckets = _storage().list_buckets()
replication = _replication()
bucket_info = []
for bucket in buckets:
existing_rule = replication.get_rule(bucket.name)
has_rule_for_peer = (
existing_rule and
existing_rule.target_connection_id == peer.connection_id
)
bucket_info.append({
"name": bucket.name,
"has_rule": has_rule_for_peer,
"existing_mode": existing_rule.mode if has_rule_for_peer else None,
"existing_target": existing_rule.target_bucket if has_rule_for_peer else None,
})
return render_template(
"replication_wizard.html",
principal=principal,
peer=peer,
connection=connection,
buckets=bucket_info,
csrf_token=generate_csrf,
)
@ui_bp.post("/sites/peers/<site_id>/replication-rules")
def create_peer_replication_rules(site_id: str):
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:*")
except IamError:
flash("Access denied", "danger")
return redirect(url_for("ui.sites_dashboard"))
registry = _site_registry()
peer = registry.get_peer(site_id)
if not peer or not peer.connection_id:
flash("Invalid peer site or no connection configured", "danger")
return redirect(url_for("ui.sites_dashboard"))
from .replication import REPLICATION_MODE_NEW_ONLY, REPLICATION_MODE_ALL
import time as time_module
selected_buckets = request.form.getlist("buckets")
mode = request.form.get("mode", REPLICATION_MODE_NEW_ONLY)
if not selected_buckets:
flash("No buckets selected", "warning")
return redirect(url_for("ui.sites_dashboard"))
created = 0
failed = 0
replication = _replication()
for bucket_name in selected_buckets:
target_bucket = request.form.get(f"target_{bucket_name}", bucket_name).strip()
if not target_bucket:
target_bucket = bucket_name
try:
rule = ReplicationRule(
bucket_name=bucket_name,
target_connection_id=peer.connection_id,
target_bucket=target_bucket,
enabled=True,
mode=mode,
created_at=time_module.time(),
)
replication.set_rule(rule)
if mode == REPLICATION_MODE_ALL:
replication.replicate_existing_objects(bucket_name)
created += 1
except Exception:
failed += 1
if created > 0:
flash(f"Created {created} replication rule(s) for {peer.display_name or peer.site_id}", "success")
if failed > 0:
flash(f"Failed to create {failed} rule(s)", "danger")
return redirect(url_for("ui.sites_dashboard"))
@ui_bp.get("/sites/peers/<site_id>/sync-stats")
def get_peer_sync_stats(site_id: str):
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:*")
except IamError:
return jsonify({"error": "Access denied"}), 403
registry = _site_registry()
peer = registry.get_peer(site_id)
if not peer:
return jsonify({"error": "Peer not found"}), 404
if not peer.connection_id:
return jsonify({"error": "No connection configured"}), 400
replication = _replication()
all_rules = replication.list_rules()
stats = {
"buckets_syncing": 0,
"objects_synced": 0,
"objects_pending": 0,
"objects_failed": 0,
"bytes_synced": 0,
"last_sync_at": None,
"buckets": [],
}
for rule in all_rules:
if rule.target_connection_id != peer.connection_id:
continue
stats["buckets_syncing"] += 1
bucket_stats = {
"bucket_name": rule.bucket_name,
"target_bucket": rule.target_bucket,
"mode": rule.mode,
"enabled": rule.enabled,
}
if rule.stats:
stats["objects_synced"] += rule.stats.objects_synced
stats["objects_pending"] += rule.stats.objects_pending
stats["bytes_synced"] += rule.stats.bytes_synced
if rule.stats.last_sync_at:
if not stats["last_sync_at"] or rule.stats.last_sync_at > stats["last_sync_at"]:
stats["last_sync_at"] = rule.stats.last_sync_at
bucket_stats["last_sync_at"] = rule.stats.last_sync_at
bucket_stats["objects_synced"] = rule.stats.objects_synced
bucket_stats["objects_pending"] = rule.stats.objects_pending
failure_count = replication.get_failure_count(rule.bucket_name)
stats["objects_failed"] += failure_count
bucket_stats["failures"] = failure_count
stats["buckets"].append(bucket_stats)
return jsonify(stats)
@ui_bp.app_errorhandler(404)
def ui_not_found(error): # type: ignore[override]
prefix = ui_bp.url_prefix or ""

View File

@@ -0,0 +1,226 @@
{% extends "base.html" %}
{% block title %}Set Up Replication - S3 Compatible Storage{% endblock %}
{% block content %}
<div class="page-header d-flex justify-content-between align-items-center mb-4">
<div>
<nav aria-label="breadcrumb">
<ol class="breadcrumb mb-1">
<li class="breadcrumb-item"><a href="{{ url_for('ui.sites_dashboard') }}">Sites</a></li>
<li class="breadcrumb-item active" aria-current="page">Replication Wizard</li>
</ol>
</nav>
<h1 class="h3 mb-1 d-flex align-items-center gap-2">
<svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
<path d="M8 4a.5.5 0 0 1 .5.5V6a.5.5 0 0 1-1 0V4.5A.5.5 0 0 1 8 4zM3.732 5.732a.5.5 0 0 1 .707 0l.915.914a.5.5 0 1 1-.708.708l-.914-.915a.5.5 0 0 1 0-.707zM2 10a.5.5 0 0 1 .5-.5h1.586a.5.5 0 0 1 0 1H2.5A.5.5 0 0 1 2 10zm9.5 0a.5.5 0 0 1 .5-.5h1.5a.5.5 0 0 1 0 1H12a.5.5 0 0 1-.5-.5zm.754-4.246a.389.389 0 0 0-.527-.02L7.547 9.31a.91.91 0 1 0 1.302 1.258l3.434-4.297a.389.389 0 0 0-.029-.518z"/>
<path fill-rule="evenodd" d="M0 10a8 8 0 1 1 15.547 2.661c-.442 1.253-1.845 1.602-2.932 1.25C11.309 13.488 9.475 13 8 13c-1.474 0-3.31.488-4.615.911-1.087.352-2.49.003-2.932-1.25A7.988 7.988 0 0 1 0 10zm8-7a7 7 0 0 0-6.603 9.329c.203.575.923.876 1.68.63C4.397 12.533 6.358 12 8 12s3.604.532 4.923.96c.757.245 1.477-.056 1.68-.631A7 7 0 0 0 8 3z"/>
</svg>
Set Up Replication
</h1>
<p class="text-muted mb-0 mt-1">Configure bucket replication to <strong>{{ peer.display_name or peer.site_id }}</strong></p>
</div>
</div>
<div class="row g-4">
<div class="col-lg-4 col-md-5">
<div class="card shadow-sm border-0 mb-4" style="border-radius: 1rem;">
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09z"/>
</svg>
Peer Site
</h5>
</div>
<div class="card-body px-4 pb-4">
<dl class="mb-0">
<dt class="text-muted small">Site ID</dt>
<dd class="mb-2">{{ peer.site_id }}</dd>
<dt class="text-muted small">Endpoint</dt>
<dd class="mb-2 text-truncate" title="{{ peer.endpoint }}">{{ peer.endpoint }}</dd>
<dt class="text-muted small">Region</dt>
<dd class="mb-2"><span class="badge bg-primary bg-opacity-10 text-primary">{{ peer.region }}</span></dd>
<dt class="text-muted small">Connection</dt>
<dd class="mb-0"><span class="badge bg-secondary bg-opacity-10 text-secondary">{{ connection.name }}</span></dd>
</dl>
</div>
</div>
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
</svg>
Replication Modes
</h5>
</div>
<div class="card-body px-4 pb-4 small">
<p class="mb-2"><strong>New Only:</strong> Only replicate new objects uploaded after the rule is created.</p>
<p class="mb-2"><strong>All Objects:</strong> Replicate all existing objects plus new uploads.</p>
<p class="mb-0"><strong>Bidirectional:</strong> Two-way sync between sites. Changes on either side are synchronized.</p>
</div>
</div>
</div>
<div class="col-lg-8 col-md-7">
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>
</svg>
Select Buckets to Replicate
</h5>
<p class="text-muted small mb-0">Choose which buckets should be replicated to this peer site</p>
</div>
<div class="card-body px-4 pb-4">
{% if buckets %}
<form method="POST" action="{{ url_for('ui.create_peer_replication_rules', site_id=peer.site_id) }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<div class="mb-4">
<label for="mode" class="form-label fw-medium">Replication Mode</label>
<select class="form-select" id="mode" name="mode">
<option value="new_only">New Objects Only</option>
<option value="all">All Objects (includes existing)</option>
<option value="bidirectional">Bidirectional Sync</option>
</select>
</div>
<div class="table-responsive">
<table class="table table-hover align-middle mb-0">
<thead class="table-light">
<tr>
<th scope="col" style="width: 40px;">
<input type="checkbox" class="form-check-input" id="selectAll">
</th>
<th scope="col">Local Bucket</th>
<th scope="col">Target Bucket Name</th>
<th scope="col">Status</th>
</tr>
</thead>
<tbody>
{% for bucket in buckets %}
<tr>
<td>
<input type="checkbox" class="form-check-input bucket-checkbox"
name="buckets" value="{{ bucket.name }}"
{% if bucket.has_rule %}disabled{% endif %}>
</td>
<td>
<div class="d-flex align-items-center gap-2">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>
</svg>
<span class="fw-medium">{{ bucket.name }}</span>
</div>
</td>
<td>
<input type="text" class="form-control form-control-sm"
name="target_{{ bucket.name }}"
value="{{ bucket.existing_target or bucket.name }}"
placeholder="{{ bucket.name }}"
{% if bucket.has_rule %}disabled{% endif %}>
</td>
<td>
{% if bucket.has_rule %}
<span class="badge bg-info bg-opacity-10 text-info">
Already configured ({{ bucket.existing_mode }})
</span>
{% else %}
<span class="badge bg-secondary bg-opacity-10 text-secondary">
Not configured
</span>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<div class="d-flex gap-2 mt-4 pt-3 border-top">
<button type="submit" class="btn btn-primary" id="submitBtn" disabled>
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
<path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
</svg>
Create Replication Rules
</button>
<a href="{{ url_for('ui.sites_dashboard') }}" class="btn btn-outline-secondary">
Skip for Now
</a>
</div>
</form>
{% else %}
<div class="empty-state text-center py-5">
<div class="empty-state-icon mx-auto mb-3">
<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" fill="currentColor" viewBox="0 0 16 16">
<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>
</svg>
</div>
<h5 class="fw-semibold mb-2">No buckets yet</h5>
<p class="text-muted mb-3">Create some buckets first, then come back to set up replication.</p>
<a href="{{ url_for('ui.buckets_overview') }}" class="btn btn-primary">
Go to Buckets
</a>
</div>
{% endif %}
</div>
</div>
</div>
</div>
<script>
(function() {
const selectAllCheckbox = document.getElementById('selectAll');
const bucketCheckboxes = document.querySelectorAll('.bucket-checkbox:not(:disabled)');
const submitBtn = document.getElementById('submitBtn');
function updateSubmitButton() {
const checkedCount = document.querySelectorAll('.bucket-checkbox:checked').length;
if (submitBtn) {
submitBtn.disabled = checkedCount === 0;
const text = checkedCount > 0
? `Create ${checkedCount} Replication Rule${checkedCount > 1 ? 's' : ''}`
: 'Create Replication Rules';
submitBtn.innerHTML = `
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
<path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
</svg>
${text}
`;
}
}
function updateSelectAll() {
if (selectAllCheckbox && bucketCheckboxes.length > 0) {
const allChecked = Array.from(bucketCheckboxes).every(cb => cb.checked);
const someChecked = Array.from(bucketCheckboxes).some(cb => cb.checked);
selectAllCheckbox.checked = allChecked;
selectAllCheckbox.indeterminate = someChecked && !allChecked;
}
}
if (selectAllCheckbox) {
selectAllCheckbox.addEventListener('change', function() {
bucketCheckboxes.forEach(cb => {
cb.checked = this.checked;
});
updateSubmitButton();
});
}
bucketCheckboxes.forEach(cb => {
cb.addEventListener('change', function() {
updateSelectAll();
updateSubmitButton();
});
});
updateSelectAll();
updateSubmitButton();
})();
</script>
{% endblock %}

View File

@@ -163,11 +163,13 @@
<th scope="col">Endpoint</th>
<th scope="col">Region</th>
<th scope="col">Priority</th>
<th scope="col">Sync Status</th>
<th scope="col" class="text-end">Actions</th>
</tr>
</thead>
<tbody>
{% for peer in peers %}
{% for item in peers_with_stats %}
{% set peer = item.peer %}
<tr data-site-id="{{ peer.site_id }}">
<td class="text-center">
<span class="peer-health-status" data-site-id="{{ peer.site_id }}" title="{% if peer.is_healthy == true %}Healthy{% elif peer.is_healthy == false %}Unhealthy{% else %}Unknown{% endif %}">
@@ -207,8 +209,37 @@
</td>
<td><span class="badge bg-primary bg-opacity-10 text-primary">{{ peer.region }}</span></td>
<td><span class="badge bg-secondary bg-opacity-10 text-secondary">{{ peer.priority }}</span></td>
<td class="sync-stats-cell" data-site-id="{{ peer.site_id }}">
{% if item.has_connection %}
<div class="d-flex align-items-center gap-2">
<span class="badge bg-primary bg-opacity-10 text-primary">{{ item.buckets_syncing }} bucket{{ 's' if item.buckets_syncing != 1 else '' }}</span>
{% if item.buckets_syncing > 0 %}
<button type="button" class="btn btn-sm btn-outline-secondary btn-load-stats py-0 px-1"
data-site-id="{{ peer.site_id }}" title="Load sync details">
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>
<path d="M8 4.466V.534a.25.25 0 0 1 .41-.192l2.36 1.966c.12.1.12.284 0 .384L8.41 4.658A.25.25 0 0 1 8 4.466z"/>
</svg>
</button>
{% endif %}
</div>
<div class="sync-stats-detail d-none mt-2 small" id="stats-{{ peer.site_id }}">
<span class="spinner-border spinner-border-sm text-muted" style="width: 12px; height: 12px;"></span>
</div>
{% else %}
<span class="text-muted small">No connection</span>
{% endif %}
</td>
<td class="text-end">
<div class="btn-group btn-group-sm" role="group">
<a href="{{ url_for('ui.replication_wizard', site_id=peer.site_id) }}"
class="btn btn-outline-primary {% if not item.has_connection %}disabled{% endif %}"
title="Set up replication">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
<path d="M11.534 7h3.932a.25.25 0 0 1 .192.41l-1.966 2.36a.25.25 0 0 1-.384 0l-1.966-2.36a.25.25 0 0 1 .192-.41zm-11 2h3.932a.25.25 0 0 0 .192-.41L2.692 6.23a.25.25 0 0 0-.384 0L.342 8.59A.25.25 0 0 0 .534 9z"/>
<path fill-rule="evenodd" d="M8 3c-1.552 0-2.94.707-3.857 1.818a.5.5 0 1 1-.771-.636A6.002 6.002 0 0 1 13.917 7H12.9A5.002 5.002 0 0 0 8 3zM3.1 9a5.002 5.002 0 0 0 8.757 2.182.5.5 0 1 1 .771.636A6.002 6.002 0 0 1 2.083 9H3.1z"/>
</svg>
</a>
<button type="button" class="btn btn-outline-secondary btn-check-health"
data-site-id="{{ peer.site_id }}"
title="Check health">
@@ -427,6 +458,42 @@
});
});
});
document.querySelectorAll('.btn-load-stats').forEach(function(btn) {
btn.addEventListener('click', function() {
const siteId = this.getAttribute('data-site-id');
const detailDiv = document.getElementById('stats-' + siteId);
if (!detailDiv) return;
detailDiv.classList.remove('d-none');
detailDiv.innerHTML = '<span class="spinner-border spinner-border-sm text-muted" style="width: 12px; height: 12px;"></span> Loading...';
fetch('/ui/sites/peers/' + encodeURIComponent(siteId) + '/sync-stats')
.then(response => response.json())
.then(data => {
if (data.error) {
detailDiv.innerHTML = '<span class="text-danger">' + data.error + '</span>';
} else {
const lastSync = data.last_sync_at
? new Date(data.last_sync_at * 1000).toLocaleString()
: 'Never';
detailDiv.innerHTML = `
<div class="d-flex flex-wrap gap-2 mb-1">
<span class="text-success"><strong>${data.objects_synced}</strong> synced</span>
<span class="text-warning"><strong>${data.objects_pending}</strong> pending</span>
<span class="text-danger"><strong>${data.objects_failed}</strong> failed</span>
</div>
<div class="text-muted" style="font-size: 0.75rem;">
Last sync: ${lastSync}
</div>
`;
}
})
.catch(err => {
detailDiv.innerHTML = '<span class="text-danger">Failed to load stats</span>';
});
});
});
})();
</script>
{% endblock %}