kqjy/MyFSIO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

Add metrics history with charts, fix percentage formatting to 2 d.p.

Browse Source
This commit is contained in:
kqjy
2026-01-16 19:57:23 +08:00
parent a3b9db544c
commit 53297abe1e
4 changed files with 435 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
app/config.py
View File

@@ -76,6 +76,9 @@ class AppConfig:
display_timezone: str
lifecycle_enabled: bool
lifecycle_interval_seconds: int
metrics_history_enabled: bool
metrics_history_retention_hours: int
metrics_history_interval_minutes: int
@classmethod
def from_env(cls, overrides: Optional[Dict[str, Any]] = None) -> "AppConfig":
@@ -172,6 +175,9 @@ class AppConfig:
kms_keys_path = Path(_get("KMS_KEYS_PATH", encryption_keys_dir / "kms_keys.json")).resolve()
default_encryption_algorithm = str(_get("DEFAULT_ENCRYPTION_ALGORITHM", "AES256"))
display_timezone = str(_get("DISPLAY_TIMEZONE", "UTC"))
metrics_history_enabled = str(_get("METRICS_HISTORY_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
metrics_history_retention_hours = int(_get("METRICS_HISTORY_RETENTION_HOURS", 24))
metrics_history_interval_minutes = int(_get("METRICS_HISTORY_INTERVAL_MINUTES", 5))
return cls(storage_root=storage_root,
max_upload_size=max_upload_size,
@@ -210,7 +216,10 @@ class AppConfig:
default_encryption_algorithm=default_encryption_algorithm,
display_timezone=display_timezone,
lifecycle_enabled=lifecycle_enabled,
lifecycle_interval_seconds=lifecycle_interval_seconds)
lifecycle_interval_seconds=lifecycle_interval_seconds,
metrics_history_enabled=metrics_history_enabled,
metrics_history_retention_hours=metrics_history_retention_hours,
metrics_history_interval_minutes=metrics_history_interval_minutes)
def validate_and_report(self) -> list[str]:
"""Validate configuration and return a list of warnings/issues.
@@ -339,4 +348,7 @@ class AppConfig:
"DISPLAY_TIMEZONE": self.display_timezone,
"LIFECYCLE_ENABLED": self.lifecycle_enabled,
"LIFECYCLE_INTERVAL_SECONDS": self.lifecycle_interval_seconds,
"METRICS_HISTORY_ENABLED": self.metrics_history_enabled,
"METRICS_HISTORY_RETENTION_HOURS": self.metrics_history_retention_hours,
"METRICS_HISTORY_INTERVAL_MINUTES": self.metrics_history_interval_minutes,
}

148
app/ui.py
View File

@@ -6,6 +6,7 @@ import uuid
import psutil
import shutil
from datetime import datetime, timezone as dt_timezone
from pathlib import Path
from typing import Any
from urllib.parse import quote, urlparse
from zoneinfo import ZoneInfo
@@ -152,6 +153,69 @@ def _format_bytes(num: int) -> str:
return f"{value:.1f} PB"
_metrics_last_save_time: float = 0.0
def _get_metrics_history_path() -> Path:
storage_root = Path(current_app.config["STORAGE_ROOT"])
return storage_root / ".myfsio.sys" / "config" / "metrics_history.json"
def _load_metrics_history() -> dict:
path = _get_metrics_history_path()
if not path.exists():
return {"history": []}
try:
return json.loads(path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return {"history": []}
def _save_metrics_snapshot(cpu_percent: float, memory_percent: float, disk_percent: float, storage_bytes: int) -> None:
global _metrics_last_save_time
if not current_app.config.get("METRICS_HISTORY_ENABLED", False):
return
import time
from datetime import datetime, timezone
interval_minutes = current_app.config.get("METRICS_HISTORY_INTERVAL_MINUTES", 5)
now_ts = time.time()
if now_ts - _metrics_last_save_time < interval_minutes * 60:
return
path = _get_metrics_history_path()
path.parent.mkdir(parents=True, exist_ok=True)
data = _load_metrics_history()
history = data.get("history", [])
retention_hours = current_app.config.get("METRICS_HISTORY_RETENTION_HOURS", 24)
now = datetime.now(timezone.utc)
snapshot = {
"timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"cpu_percent": round(cpu_percent, 2),
"memory_percent": round(memory_percent, 2),
"disk_percent": round(disk_percent, 2),
"storage_bytes": storage_bytes,
}
history.append(snapshot)
cutoff = now.timestamp() - (retention_hours * 3600)
history = [
h for h in history
if datetime.fromisoformat(h["timestamp"].replace("Z", "+00:00")).timestamp() > cutoff
]
data["history"] = history
try:
path.write_text(json.dumps(data, indent=2), encoding="utf-8")
_metrics_last_save_time = now_ts
except OSError:
pass
def _friendly_error_message(exc: Exception) -> str:
message = str(exc) or "An unexpected error occurred"
if isinstance(exc, IamError):
@@ -2101,18 +2165,18 @@ def metrics_dashboard():
return render_template(
"metrics.html",
principal=principal,
cpu_percent=cpu_percent,
cpu_percent=round(cpu_percent, 2),
memory={
"total": _format_bytes(memory.total),
"available": _format_bytes(memory.available),
"used": _format_bytes(memory.used),
"percent": memory.percent,
"percent": round(memory.percent, 2),
},
disk={
"total": _format_bytes(disk.total),
"free": _format_bytes(disk.free),
"used": _format_bytes(disk.used),
"percent": disk.percent,
"percent": round(disk.percent, 2),
},
app={
"buckets": total_buckets,
@@ -2122,7 +2186,8 @@ def metrics_dashboard():
"storage_raw": total_bytes_used,
"version": APP_VERSION,
"uptime_days": uptime_days,
}
},
metrics_history_enabled=current_app.config.get("METRICS_HISTORY_ENABLED", False),
)
@@ -2162,19 +2227,21 @@ def metrics_api():
uptime_seconds = time.time() - boot_time
uptime_days = int(uptime_seconds / 86400)
_save_metrics_snapshot(cpu_percent, memory.percent, disk.percent, total_bytes_used)
return jsonify({
"cpu_percent": cpu_percent,
"cpu_percent": round(cpu_percent, 2),
"memory": {
"total": _format_bytes(memory.total),
"available": _format_bytes(memory.available),
"used": _format_bytes(memory.used),
"percent": memory.percent,
"percent": round(memory.percent, 2),
},
"disk": {
"total": _format_bytes(disk.total),
"free": _format_bytes(disk.free),
"used": _format_bytes(disk.used),
"percent": disk.percent,
"percent": round(disk.percent, 2),
},
"app": {
"buckets": total_buckets,
@@ -2187,6 +2254,73 @@ def metrics_api():
})
@ui_bp.route("/metrics/history")
def metrics_history():
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:list_users")
except IamError:
return jsonify({"error": "Access denied"}), 403
if not current_app.config.get("METRICS_HISTORY_ENABLED", False):
return jsonify({"enabled": False, "history": []})
hours = request.args.get("hours", type=int)
if hours is None:
hours = current_app.config.get("METRICS_HISTORY_RETENTION_HOURS", 24)
data = _load_metrics_history()
history = data.get("history", [])
if hours:
from datetime import datetime, timezone
cutoff = datetime.now(timezone.utc).timestamp() - (hours * 3600)
history = [
h for h in history
if datetime.fromisoformat(h["timestamp"].replace("Z", "+00:00")).timestamp() > cutoff
]
return jsonify({
"enabled": True,
"retention_hours": current_app.config.get("METRICS_HISTORY_RETENTION_HOURS", 24),
"interval_minutes": current_app.config.get("METRICS_HISTORY_INTERVAL_MINUTES", 5),
"history": history,
})
@ui_bp.route("/metrics/settings", methods=["GET", "PUT"])
def metrics_settings():
principal = _current_principal()
try:
_iam().authorize(principal, None, "iam:list_users")
except IamError:
return jsonify({"error": "Access denied"}), 403
if request.method == "GET":
return jsonify({
"enabled": current_app.config.get("METRICS_HISTORY_ENABLED", False),
"retention_hours": current_app.config.get("METRICS_HISTORY_RETENTION_HOURS", 24),
"interval_minutes": current_app.config.get("METRICS_HISTORY_INTERVAL_MINUTES", 5),
})
data = request.get_json() or {}
if "enabled" in data:
current_app.config["METRICS_HISTORY_ENABLED"] = bool(data["enabled"])
if "retention_hours" in data:
current_app.config["METRICS_HISTORY_RETENTION_HOURS"] = max(1, int(data["retention_hours"]))
if "interval_minutes" in data:
current_app.config["METRICS_HISTORY_INTERVAL_MINUTES"] = max(1, int(data["interval_minutes"]))
return jsonify({
"enabled": current_app.config.get("METRICS_HISTORY_ENABLED", False),
"retention_hours": current_app.config.get("METRICS_HISTORY_RETENTION_HOURS", 24),
"interval_minutes": current_app.config.get("METRICS_HISTORY_INTERVAL_MINUTES", 5),
})
@ui_bp.route("/buckets/<bucket_name>/lifecycle", methods=["GET", "POST", "DELETE"])
def bucket_lifecycle(bucket_name: str):
principal = _current_principal()

106
templates/docs.html
View File

@@ -39,6 +39,7 @@
<li><a href="#quotas">Bucket Quotas</a></li>
<li><a href="#encryption">Encryption</a></li>
<li><a href="#lifecycle">Lifecycle Rules</a></li>
<li><a href="#metrics">Metrics History</a></li>
<li><a href="#troubleshooting">Troubleshooting</a></li>
</ul>
</div>
@@ -181,6 +182,24 @@ python run.py --mode ui
<td><code>true</code></td>
<td>Enable file logging.</td>
</tr>
<tr class="table-secondary">
<td colspan="3" class="fw-semibold">Metrics History Settings</td>
</tr>
<tr>
<td><code>METRICS_HISTORY_ENABLED</code></td>
<td><code>false</code></td>
<td>Enable metrics history recording and charts (opt-in).</td>
</tr>
<tr>
<td><code>METRICS_HISTORY_RETENTION_HOURS</code></td>
<td><code>24</code></td>
<td>How long to retain metrics history data.</td>
</tr>
<tr>
<td><code>METRICS_HISTORY_INTERVAL_MINUTES</code></td>
<td><code>5</code></td>
<td>Interval between history snapshots.</td>
</tr>
</tbody>
</table>
</div>
@@ -976,10 +995,94 @@ curl "{{ api_base }}/&lt;bucket&gt;?lifecycle" \
</div>
</div>
</article>
<article id="troubleshooting" class="card shadow-sm docs-section">
<article id="metrics" class="card shadow-sm docs-section">
<div class="card-body">
<div class="d-flex align-items-center gap-2 mb-3">
<span class="docs-section-kicker">13</span>
<h2 class="h4 mb-0">Metrics History</h2>
</div>
<p class="text-muted">Track CPU, memory, and disk usage over time with optional metrics history. Disabled by default to minimize overhead.</p>
<h3 class="h6 text-uppercase text-muted mt-4">Enabling Metrics History</h3>
<p class="small text-muted">Set the environment variable to opt-in:</p>
<pre class="mb-3"><code class="language-bash"># PowerShell
$env:METRICS_HISTORY_ENABLED = "true"
python run.py
# Bash
export METRICS_HISTORY_ENABLED=true
python run.py</code></pre>
<h3 class="h6 text-uppercase text-muted mt-4">Configuration Options</h3>
<div class="table-responsive mb-3">
<table class="table table-sm table-bordered small">
<thead class="table-light">
<tr>
<th>Variable</th>
<th>Default</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>METRICS_HISTORY_ENABLED</code></td>
<td><code>false</code></td>
<td>Enable/disable metrics history recording</td>
</tr>
<tr>
<td><code>METRICS_HISTORY_RETENTION_HOURS</code></td>
<td><code>24</code></td>
<td>How long to keep history data (hours)</td>
</tr>
<tr>
<td><code>METRICS_HISTORY_INTERVAL_MINUTES</code></td>
<td><code>5</code></td>
<td>Interval between snapshots (minutes)</td>
</tr>
</tbody>
</table>
</div>
<h3 class="h6 text-uppercase text-muted mt-4">API Endpoints</h3>
<pre class="mb-3"><code class="language-bash"># Get metrics history (last 24 hours by default)
curl "{{ api_base | replace('/api', '/ui') }}/metrics/history" \
-H "X-Access-Key: &lt;key&gt;" -H "X-Secret-Key: &lt;secret&gt;"
# Get history for specific time range
curl "{{ api_base | replace('/api', '/ui') }}/metrics/history?hours=6" \
-H "X-Access-Key: &lt;key&gt;" -H "X-Secret-Key: &lt;secret&gt;"
# Get current settings
curl "{{ api_base | replace('/api', '/ui') }}/metrics/settings" \
-H "X-Access-Key: &lt;key&gt;" -H "X-Secret-Key: &lt;secret&gt;"
# Update settings at runtime
curl -X PUT "{{ api_base | replace('/api', '/ui') }}/metrics/settings" \
-H "Content-Type: application/json" \
-H "X-Access-Key: &lt;key&gt;" -H "X-Secret-Key: &lt;secret&gt;" \
-d '{"enabled": true, "retention_hours": 48, "interval_minutes": 10}'</code></pre>
<h3 class="h6 text-uppercase text-muted mt-4">Storage Location</h3>
<p class="small text-muted mb-3">History data is stored at:</p>
<code class="d-block mb-3">data/.myfsio.sys/config/metrics_history.json</code>
<div class="alert alert-light border mb-0">
<div class="d-flex gap-2">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
</svg>
<div>
<strong>UI Charts:</strong> When enabled, the Metrics dashboard displays line charts showing CPU, memory, and disk usage trends with a time range selector (1h, 6h, 24h, 7d).
</div>
</div>
</div>
</div>
</article>
<article id="troubleshooting" class="card shadow-sm docs-section">
<div class="card-body">
<div class="d-flex align-items-center gap-2 mb-3">
<span class="docs-section-kicker">14</span>
<h2 class="h4 mb-0">Troubleshooting &amp; tips</h2>
</div>
<div class="table-responsive">
@@ -1045,6 +1148,7 @@ curl "{{ api_base }}/&lt;bucket&gt;?lifecycle" \
<li><a href="#quotas">Bucket Quotas</a></li>
<li><a href="#encryption">Encryption</a></li>
<li><a href="#lifecycle">Lifecycle Rules</a></li>
<li><a href="#metrics">Metrics History</a></li>
<li><a href="#troubleshooting">Troubleshooting</a></li>
</ul>
<div class="docs-sidebar-callouts">

179
templates/metrics.html
View File

@@ -267,9 +267,49 @@
</div>
</div>
</div>
{% if metrics_history_enabled %}
<div class="row g-4 mt-2">
<div class="col-12">
<div class="card shadow-sm border-0">
<div class="card-header bg-transparent border-0 pt-4 px-4 d-flex justify-content-between align-items-center">
<h5 class="card-title mb-0 fw-semibold">Metrics History</h5>
<div class="d-flex gap-2 align-items-center">
<select class="form-select form-select-sm" id="historyTimeRange" style="width: auto;">
<option value="1">Last 1 hour</option>
<option value="6">Last 6 hours</option>
<option value="24" selected>Last 24 hours</option>
<option value="168">Last 7 days</option>
</select>
</div>
</div>
<div class="card-body p-4">
<div class="row">
<div class="col-md-4 mb-4">
<h6 class="text-muted small fw-bold text-uppercase mb-3">CPU Usage</h6>
<canvas id="cpuHistoryChart" height="200"></canvas>
</div>
<div class="col-md-4 mb-4">
<h6 class="text-muted small fw-bold text-uppercase mb-3">Memory Usage</h6>
<canvas id="memoryHistoryChart" height="200"></canvas>
</div>
<div class="col-md-4 mb-4">
<h6 class="text-muted small fw-bold text-uppercase mb-3">Disk Usage</h6>
<canvas id="diskHistoryChart" height="200"></canvas>
</div>
</div>
<p class="text-muted small mb-0 text-center" id="historyStatus">Loading history data...</p>
</div>
</div>
</div>
</div>
{% endif %}
{% endblock %}
{% block extra_scripts %}
{% if metrics_history_enabled %}
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
{% endif %}
<script>
(function() {
var refreshInterval = 5000;
@@ -285,7 +325,7 @@
.then(function(data) {
var el;
el = document.querySelector('[data-metric="cpu_percent"]');
if (el) el.textContent = data.cpu_percent;
if (el) el.textContent = data.cpu_percent.toFixed(2);
el = document.querySelector('[data-metric="cpu_bar"]');
if (el) {
el.style.width = data.cpu_percent + '%';
@@ -298,7 +338,7 @@
}
el = document.querySelector('[data-metric="memory_percent"]');
if (el) el.textContent = data.memory.percent;
if (el) el.textContent = data.memory.percent.toFixed(2);
el = document.querySelector('[data-metric="memory_bar"]');
if (el) el.style.width = data.memory.percent + '%';
el = document.querySelector('[data-metric="memory_used"]');
@@ -307,7 +347,7 @@
if (el) el.textContent = data.memory.total;
el = document.querySelector('[data-metric="disk_percent"]');
if (el) el.textContent = data.disk.percent;
if (el) el.textContent = data.disk.percent.toFixed(2);
el = document.querySelector('[data-metric="disk_bar"]');
if (el) {
el.style.width = data.disk.percent + '%';
@@ -372,5 +412,138 @@
startPolling();
})();
{% if metrics_history_enabled %}
(function() {
var cpuChart = null;
var memoryChart = null;
var diskChart = null;
var historyStatus = document.getElementById('historyStatus');
var timeRangeSelect = document.getElementById('historyTimeRange');
var historyTimer = null;
var MAX_DATA_POINTS = 500;
function createChart(ctx, label, color) {
return new Chart(ctx, {
type: 'line',
data: {
labels: [],
datasets: [{
label: label,
data: [],
borderColor: color,
backgroundColor: color + '20',
fill: true,
tension: 0.3,
pointRadius: 3,
pointHoverRadius: 6,
hitRadius: 10,
}]
},
options: {
responsive: true,
maintainAspectRatio: true,
animation: false,
plugins: {
legend: { display: false },
tooltip: {
callbacks: {
label: function(ctx) { return ctx.parsed.y.toFixed(2) + '%'; }
}
}
},
scales: {
x: {
display: true,
ticks: { maxTicksAuto: true, maxRotation: 0, font: { size: 10 }, autoSkip: true, maxTicksLimit: 10 }
},
y: {
display: true,
min: 0,
max: 100,
ticks: { callback: function(v) { return v + '%'; } }
}
}
}
});
}
function initCharts() {
var cpuCtx = document.getElementById('cpuHistoryChart');
var memCtx = document.getElementById('memoryHistoryChart');
var diskCtx = document.getElementById('diskHistoryChart');
if (cpuCtx) cpuChart = createChart(cpuCtx, 'CPU %', '#0d6efd');
if (memCtx) memoryChart = createChart(memCtx, 'Memory %', '#0dcaf0');
if (diskCtx) diskChart = createChart(diskCtx, 'Disk %', '#ffc107');
}
function formatTime(ts) {
var d = new Date(ts);
return d.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
}
function loadHistory() {
if (document.hidden) return;
var hours = timeRangeSelect ? timeRangeSelect.value : 24;
fetch('/ui/metrics/history?hours=' + hours)
.then(function(r) { return r.json(); })
.then(function(data) {
if (!data.enabled || !data.history || data.history.length === 0) {
if (historyStatus) historyStatus.textContent = 'No history data available yet. Data is recorded every ' + (data.interval_minutes || 5) + ' minutes.';
return;
}
var history = data.history.slice(-MAX_DATA_POINTS);
var labels = history.map(function(h) { return formatTime(h.timestamp); });
var cpuData = history.map(function(h) { return h.cpu_percent; });
var memData = history.map(function(h) { return h.memory_percent; });
var diskData = history.map(function(h) { return h.disk_percent; });
if (cpuChart) {
cpuChart.data.labels = labels;
cpuChart.data.datasets[0].data = cpuData;
cpuChart.update('none');
}
if (memoryChart) {
memoryChart.data.labels = labels;
memoryChart.data.datasets[0].data = memData;
memoryChart.update('none');
}
if (diskChart) {
diskChart.data.labels = labels;
diskChart.data.datasets[0].data = diskData;
diskChart.update('none');
}
if (historyStatus) historyStatus.textContent = 'Showing ' + history.length + ' data points';
})
.catch(function(err) {
console.error('History fetch error:', err);
if (historyStatus) historyStatus.textContent = 'Failed to load history data';
});
}
function startHistoryPolling() {
if (historyTimer) clearInterval(historyTimer);
historyTimer = setInterval(loadHistory, 60000);
}
if (timeRangeSelect) {
timeRangeSelect.addEventListener('change', loadHistory);
}
document.addEventListener('visibilitychange', function() {
if (document.hidden) {
if (historyTimer) clearInterval(historyTimer);
historyTimer = null;
} else {
loadHistory();
startHistoryPolling();
}
});
initCharts();
loadHistory();
startHistoryPolling();
})();
{% endif %}
</script>
{% endblock %}