Add replication failure tracking and lifecycle execution history
This commit is contained in:
@@ -1184,11 +1184,65 @@
|
||||
<path d="M8 3.5a.5.5 0 0 0-1 0V9a.5.5 0 0 0 .252.434l3.5 2a.5.5 0 0 0 .496-.868L8 8.71V3.5z"/>
|
||||
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm7-8A7 7 0 1 1 1 8a7 7 0 0 1 14 0z"/>
|
||||
</svg>
|
||||
<strong>Last sync:</strong>
|
||||
<strong>Last sync:</strong>
|
||||
<span data-stat="last-sync-time"></span>
|
||||
<span data-stat="last-sync-key"></span>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="card border mb-4" id="replication-failures-card" style="display: none;"
|
||||
data-failures-endpoint="{{ url_for('ui.get_replication_failures', bucket_name=bucket_name) }}"
|
||||
data-retry-endpoint="{{ url_for('ui.retry_replication_failure', bucket_name=bucket_name, object_key='__KEY__') }}"
|
||||
data-retry-all-endpoint="{{ url_for('ui.retry_all_replication_failures', bucket_name=bucket_name) }}"
|
||||
data-dismiss-endpoint="{{ url_for('ui.dismiss_replication_failure', bucket_name=bucket_name, object_key='__KEY__') }}"
|
||||
data-clear-endpoint="{{ url_for('ui.clear_replication_failures', bucket_name=bucket_name) }}">
|
||||
<div class="card-header d-flex justify-content-between align-items-center py-2">
|
||||
<div class="d-flex align-items-center">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-danger me-2" viewBox="0 0 16 16">
|
||||
<path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/>
|
||||
</svg>
|
||||
<span class="fw-semibold small">Failed Replications</span>
|
||||
<span class="badge bg-danger ms-2" id="replication-failure-count">0</span>
|
||||
</div>
|
||||
<div class="btn-group btn-group-sm">
|
||||
<button class="btn btn-outline-primary btn-sm" id="retry-all-failures-btn" title="Retry All">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 3a5 5 0 1 1-4.546 2.914.5.5 0 0 0-.908-.417A6 6 0 1 0 8 2v1z"/>
|
||||
<path d="M8 4.466V.534a.25.25 0 0 0-.41-.192L5.23 2.308a.25.25 0 0 0 0 .384l2.36 1.966A.25.25 0 0 0 8 4.466z"/>
|
||||
</svg>
|
||||
Retry All
|
||||
</button>
|
||||
<button class="btn btn-outline-secondary btn-sm" id="clear-failures-btn" title="Clear All">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Clear
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body p-0">
|
||||
<div class="table-responsive">
|
||||
<table class="table table-sm table-hover mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th class="ps-3">Object Key</th>
|
||||
<th>Error</th>
|
||||
<th>Last Attempt</th>
|
||||
<th class="text-center">Attempts</th>
|
||||
<th class="text-end pe-3">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="replication-failures-body">
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="px-3 py-2 border-top" id="replication-failures-pagination" style="display: none;">
|
||||
<button class="btn btn-link btn-sm p-0" id="show-more-failures">Show more...</button>
|
||||
<span class="text-muted small ms-2" id="failures-shown-count"></span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h6 class="text-muted text-uppercase small mb-3">Replication Target</h6>
|
||||
<div class="card border mb-4">
|
||||
<div class="card-body">
|
||||
@@ -1526,6 +1580,46 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card shadow-sm mt-4" id="lifecycle-history-card"
|
||||
data-history-endpoint="{{ url_for('ui.get_lifecycle_history', bucket_name=bucket_name) }}">
|
||||
<div class="card-header d-flex align-items-center">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" fill="currentColor" class="text-primary me-2" viewBox="0 0 16 16">
|
||||
<path d="M8.515 1.019A7 7 0 0 0 8 1V0a8 8 0 0 1 .589.022l-.074.997zm2.004.45a7.003 7.003 0 0 0-.985-.299l.219-.976c.383.086.76.2 1.126.342l-.36.933zm1.37.71a7.01 7.01 0 0 0-.439-.27l.493-.87a8.025 8.025 0 0 1 .979.654l-.615.789a6.996 6.996 0 0 0-.418-.302zm1.834 1.79a6.99 6.99 0 0 0-.653-.796l.724-.69c.27.285.52.59.747.91l-.818.576zm.744 1.352a7.08 7.08 0 0 0-.214-.468l.893-.45a7.976 7.976 0 0 1 .45 1.088l-.95.313a7.023 7.023 0 0 0-.179-.483zm.53 2.507a6.991 6.991 0 0 0-.1-1.025l.985-.17c.067.386.106.778.116 1.17l-1 .025zm-.131 1.538c.033-.17.06-.339.081-.51l.993.123a7.957 7.957 0 0 1-.23 1.155l-.964-.267c.046-.165.086-.332.12-.501zm-.952 2.379c.184-.29.346-.594.486-.908l.914.405c-.16.36-.345.706-.555 1.038l-.845-.535zm-.964 1.205c.122-.122.239-.248.35-.378l.758.653a8.073 8.073 0 0 1-.401.432l-.707-.707z"/>
|
||||
<path d="M8 1a7 7 0 1 0 4.95 11.95l.707.707A8.001 8.001 0 1 1 8 0v1z"/>
|
||||
<path d="M7.5 3a.5.5 0 0 1 .5.5v5.21l3.248 1.856a.5.5 0 0 1-.496.868l-3.5-2A.5.5 0 0 1 7 9V3.5a.5.5 0 0 1 .5-.5z"/>
|
||||
</svg>
|
||||
<span class="fw-semibold">Execution History</span>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<p class="text-muted small mb-3">Lifecycle rules are evaluated automatically (default: every hour). Recent executions are shown below.</p>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-sm align-middle mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>Executed</th>
|
||||
<th class="text-center">Deleted</th>
|
||||
<th class="text-center">Versions</th>
|
||||
<th class="text-center">Aborted</th>
|
||||
<th class="text-center">Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="lifecycle-history-body">
|
||||
<tr>
|
||||
<td colspan="5" class="text-center text-muted py-4">
|
||||
<div class="spinner-border spinner-border-sm me-2" role="status"></div>
|
||||
Loading...
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="mt-2" id="lifecycle-history-pagination" style="display: none;">
|
||||
<button class="btn btn-link btn-sm p-0" id="show-more-history">Show more...</button>
|
||||
<span class="text-muted small ms-2" id="history-shown-count"></span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-4">
|
||||
<div class="card bg-body-tertiary border-0 mb-3">
|
||||
@@ -5018,7 +5112,139 @@
|
||||
if (orphanedEl) orphanedEl.innerHTML = '<span class="spinner-border spinner-border-sm" role="status"></span>';
|
||||
if (bytesEl) bytesEl.innerHTML = '<span class="spinner-border spinner-border-sm" role="status"></span>';
|
||||
loadReplicationStats();
|
||||
loadReplicationFailures();
|
||||
});
|
||||
|
||||
const failuresCard = document.getElementById('replication-failures-card');
|
||||
const failuresBody = document.getElementById('replication-failures-body');
|
||||
const failureCountBadge = document.getElementById('replication-failure-count');
|
||||
const retryAllBtn = document.getElementById('retry-all-failures-btn');
|
||||
const clearFailuresBtn = document.getElementById('clear-failures-btn');
|
||||
const showMoreFailuresBtn = document.getElementById('show-more-failures');
|
||||
const failuresPagination = document.getElementById('replication-failures-pagination');
|
||||
const failuresShownCount = document.getElementById('failures-shown-count');
|
||||
|
||||
let failuresExpanded = false;
|
||||
let currentFailures = [];
|
||||
|
||||
const loadReplicationFailures = async () => {
|
||||
if (!failuresCard) return;
|
||||
|
||||
const endpoint = failuresCard.dataset.failuresEndpoint;
|
||||
const limit = failuresExpanded ? 50 : 5;
|
||||
|
||||
try {
|
||||
const resp = await fetch(`${endpoint}?limit=${limit}`);
|
||||
if (!resp.ok) throw new Error('Failed to fetch failures');
|
||||
const data = await resp.json();
|
||||
|
||||
currentFailures = data.failures;
|
||||
const total = data.total;
|
||||
|
||||
if (total > 0) {
|
||||
failuresCard.style.display = '';
|
||||
failureCountBadge.textContent = total;
|
||||
renderFailures(currentFailures);
|
||||
|
||||
if (total > 5 && !failuresExpanded) {
|
||||
failuresPagination.style.display = '';
|
||||
failuresShownCount.textContent = `Showing ${Math.min(5, total)} of ${total}`;
|
||||
} else {
|
||||
failuresPagination.style.display = 'none';
|
||||
}
|
||||
} else {
|
||||
failuresCard.style.display = 'none';
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to load replication failures:', err);
|
||||
}
|
||||
};
|
||||
|
||||
const renderFailures = (failures) => {
|
||||
if (!failuresBody) return;
|
||||
failuresBody.innerHTML = failures.map(f => `
|
||||
<tr>
|
||||
<td class="ps-3 text-break" style="max-width: 200px;">
|
||||
<code class="small">${escapeHtml(f.object_key)}</code>
|
||||
</td>
|
||||
<td class="small text-muted text-break" style="max-width: 250px;" title="${escapeHtml(f.error_message)}">
|
||||
${escapeHtml(f.error_message.length > 60 ? f.error_message.substring(0, 60) + '...' : f.error_message)}
|
||||
</td>
|
||||
<td class="small text-muted">${new Date(f.timestamp * 1000).toLocaleString()}</td>
|
||||
<td class="text-center"><span class="badge bg-secondary">${f.failure_count}</span></td>
|
||||
<td class="text-end pe-3">
|
||||
<button class="btn btn-sm btn-outline-primary py-0 px-2" onclick="retryFailure('${escapeHtml(f.object_key)}')" title="Retry">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 3a5 5 0 1 1-4.546 2.914.5.5 0 0 0-.908-.417A6 6 0 1 0 8 2v1z"/>
|
||||
<path d="M8 4.466V.534a.25.25 0 0 0-.41-.192L5.23 2.308a.25.25 0 0 0 0 .384l2.36 1.966A.25.25 0 0 0 8 4.466z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button class="btn btn-sm btn-outline-secondary py-0 px-2" onclick="dismissFailure('${escapeHtml(f.object_key)}')" title="Dismiss">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
|
||||
</svg>
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
`).join('');
|
||||
};
|
||||
|
||||
window.retryFailure = async (objectKey) => {
|
||||
const endpoint = failuresCard.dataset.retryEndpoint.replace('__KEY__', encodeURIComponent(objectKey));
|
||||
try {
|
||||
const resp = await fetch(endpoint, { method: 'POST' });
|
||||
if (resp.ok) {
|
||||
loadReplicationFailures();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to retry:', err);
|
||||
}
|
||||
};
|
||||
|
||||
window.dismissFailure = async (objectKey) => {
|
||||
const endpoint = failuresCard.dataset.dismissEndpoint.replace('__KEY__', encodeURIComponent(objectKey));
|
||||
try {
|
||||
const resp = await fetch(endpoint, { method: 'DELETE' });
|
||||
if (resp.ok) {
|
||||
loadReplicationFailures();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to dismiss:', err);
|
||||
}
|
||||
};
|
||||
|
||||
retryAllBtn?.addEventListener('click', async () => {
|
||||
const endpoint = failuresCard.dataset.retryAllEndpoint;
|
||||
try {
|
||||
const resp = await fetch(endpoint, { method: 'POST' });
|
||||
if (resp.ok) {
|
||||
loadReplicationFailures();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to retry all:', err);
|
||||
}
|
||||
});
|
||||
|
||||
clearFailuresBtn?.addEventListener('click', async () => {
|
||||
if (!confirm('Clear all failure records?')) return;
|
||||
const endpoint = failuresCard.dataset.clearEndpoint;
|
||||
try {
|
||||
const resp = await fetch(endpoint, { method: 'DELETE' });
|
||||
if (resp.ok) {
|
||||
loadReplicationFailures();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to clear failures:', err);
|
||||
}
|
||||
});
|
||||
|
||||
showMoreFailuresBtn?.addEventListener('click', () => {
|
||||
failuresExpanded = !failuresExpanded;
|
||||
showMoreFailuresBtn.textContent = failuresExpanded ? 'Show less' : 'Show more...';
|
||||
loadReplicationFailures();
|
||||
});
|
||||
|
||||
loadReplicationFailures();
|
||||
}
|
||||
|
||||
const algoAes256Radio = document.getElementById('algo_aes256');
|
||||
@@ -5657,6 +5883,83 @@
|
||||
};
|
||||
|
||||
if (lifecycleCard) loadLifecycleRules();
|
||||
|
||||
const lifecycleHistoryCard = document.getElementById('lifecycle-history-card');
|
||||
const lifecycleHistoryBody = document.getElementById('lifecycle-history-body');
|
||||
const lifecycleHistoryPagination = document.getElementById('lifecycle-history-pagination');
|
||||
const showMoreHistoryBtn = document.getElementById('show-more-history');
|
||||
const historyShownCount = document.getElementById('history-shown-count');
|
||||
let historyExpanded = false;
|
||||
|
||||
const loadLifecycleHistory = async () => {
|
||||
if (!lifecycleHistoryCard || !lifecycleHistoryBody) return;
|
||||
|
||||
const endpoint = lifecycleHistoryCard.dataset.historyEndpoint;
|
||||
const limit = historyExpanded ? 50 : 5;
|
||||
|
||||
lifecycleHistoryBody.innerHTML = '<tr><td colspan="5" class="text-center text-muted py-4"><div class="spinner-border spinner-border-sm me-2" role="status"></div>Loading...</td></tr>';
|
||||
|
||||
try {
|
||||
const resp = await fetch(`${endpoint}?limit=${limit}`);
|
||||
if (!resp.ok) throw new Error('Failed to fetch history');
|
||||
const data = await resp.json();
|
||||
|
||||
if (!data.enabled) {
|
||||
lifecycleHistoryBody.innerHTML = '<tr><td colspan="5" class="text-center text-muted py-4">Lifecycle enforcement is not enabled</td></tr>';
|
||||
return;
|
||||
}
|
||||
|
||||
const executions = data.executions || [];
|
||||
const total = data.total || 0;
|
||||
|
||||
if (executions.length === 0) {
|
||||
lifecycleHistoryBody.innerHTML = '<tr><td colspan="5" class="text-center text-muted py-4">No executions recorded yet</td></tr>';
|
||||
lifecycleHistoryPagination.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
lifecycleHistoryBody.innerHTML = executions.map(e => {
|
||||
const date = new Date(e.timestamp * 1000);
|
||||
const hasErrors = e.errors && e.errors.length > 0;
|
||||
const hasActivity = e.objects_deleted > 0 || e.versions_deleted > 0 || e.uploads_aborted > 0;
|
||||
let statusBadge;
|
||||
if (hasErrors) {
|
||||
statusBadge = '<span class="badge bg-danger">Errors</span>';
|
||||
} else if (hasActivity) {
|
||||
statusBadge = '<span class="badge bg-success">Success</span>';
|
||||
} else {
|
||||
statusBadge = '<span class="badge bg-secondary">No action</span>';
|
||||
}
|
||||
const errorTooltip = hasErrors ? ` title="${escapeHtml(e.errors.join('; '))}"` : '';
|
||||
return `<tr${errorTooltip}>
|
||||
<td class="small">${date.toLocaleString()}</td>
|
||||
<td class="text-center"><span class="badge bg-danger-subtle text-danger">${e.objects_deleted}</span></td>
|
||||
<td class="text-center"><span class="badge bg-warning-subtle text-warning">${e.versions_deleted}</span></td>
|
||||
<td class="text-center"><span class="badge bg-secondary">${e.uploads_aborted}</span></td>
|
||||
<td class="text-center">${statusBadge}</td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
|
||||
if (total > 5 && !historyExpanded) {
|
||||
lifecycleHistoryPagination.style.display = '';
|
||||
historyShownCount.textContent = `Showing ${Math.min(5, total)} of ${total}`;
|
||||
} else {
|
||||
lifecycleHistoryPagination.style.display = 'none';
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to load lifecycle history:', err);
|
||||
lifecycleHistoryBody.innerHTML = '<tr><td colspan="5" class="text-center text-danger py-4">Failed to load history</td></tr>';
|
||||
}
|
||||
};
|
||||
|
||||
showMoreHistoryBtn?.addEventListener('click', () => {
|
||||
historyExpanded = !historyExpanded;
|
||||
showMoreHistoryBtn.textContent = historyExpanded ? 'Show less' : 'Show more...';
|
||||
loadLifecycleHistory();
|
||||
});
|
||||
|
||||
if (lifecycleHistoryCard) loadLifecycleHistory();
|
||||
|
||||
if (corsCard) loadCorsRules();
|
||||
if (aclCard) loadAcl();
|
||||
</script>
|
||||
|
||||
Reference in New Issue
Block a user