diff --git a/app/operation_metrics.py b/app/operation_metrics.py index 67a63c2..3a002e1 100644 --- a/app/operation_metrics.py +++ b/app/operation_metrics.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import logging +import random import threading import time from dataclasses import dataclass, field @@ -9,6 +10,8 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional +MAX_LATENCY_SAMPLES = 5000 + logger = logging.getLogger(__name__) @@ -22,6 +25,17 @@ class OperationStats: latency_max_ms: float = 0.0 bytes_in: int = 0 bytes_out: int = 0 + latency_samples: List[float] = field(default_factory=list) + + @staticmethod + def _compute_percentile(sorted_data: List[float], p: float) -> float: + if not sorted_data: + return 0.0 + k = (len(sorted_data) - 1) * (p / 100.0) + f = int(k) + c = min(f + 1, len(sorted_data) - 1) + d = k - f + return sorted_data[f] + d * (sorted_data[c] - sorted_data[f]) def record(self, latency_ms: float, success: bool, bytes_in: int = 0, bytes_out: int = 0) -> None: self.count += 1 @@ -36,10 +50,17 @@ class OperationStats: self.latency_max_ms = latency_ms self.bytes_in += bytes_in self.bytes_out += bytes_out + if len(self.latency_samples) < MAX_LATENCY_SAMPLES: + self.latency_samples.append(latency_ms) + else: + j = random.randint(0, self.count - 1) + if j < MAX_LATENCY_SAMPLES: + self.latency_samples[j] = latency_ms def to_dict(self) -> Dict[str, Any]: avg_latency = self.latency_sum_ms / self.count if self.count > 0 else 0.0 min_latency = self.latency_min_ms if self.latency_min_ms != float("inf") else 0.0 + sorted_latencies = sorted(self.latency_samples) return { "count": self.count, "success_count": self.success_count, @@ -47,6 +68,9 @@ class OperationStats: "latency_avg_ms": round(avg_latency, 2), "latency_min_ms": round(min_latency, 2), "latency_max_ms": round(self.latency_max_ms, 2), + "latency_p50_ms": round(self._compute_percentile(sorted_latencies, 50), 2), + "latency_p95_ms": round(self._compute_percentile(sorted_latencies, 95), 2), + "latency_p99_ms": round(self._compute_percentile(sorted_latencies, 99), 2), "bytes_in": self.bytes_in, "bytes_out": self.bytes_out, } @@ -62,6 +86,11 @@ class OperationStats: self.latency_max_ms = other.latency_max_ms self.bytes_in += other.bytes_in self.bytes_out += other.bytes_out + combined = self.latency_samples + other.latency_samples + if len(combined) > MAX_LATENCY_SAMPLES: + random.shuffle(combined) + combined = combined[:MAX_LATENCY_SAMPLES] + self.latency_samples = combined @dataclass diff --git a/templates/metrics.html b/templates/metrics.html index 659729c..56a41d1 100644 --- a/templates/metrics.html +++ b/templates/metrics.html @@ -2,20 +2,40 @@ {% block content %}
-

System Metrics

-

Real-time server performance and storage usage

+

Metrics

+

Server performance, API operations, and historical trends

-
- - - Auto-refresh: 5s +
+ + + 5s - + +
+
@@ -28,250 +48,170 @@ -
-
-
-
-
-
CPU Usage
-
- - - + + +
+ +
+
+
+
+
+
+
+ + + +
+
CPU Usage
+
+ +
+
+ Normal +
+
-

{{ cpu_percent }}%

-
-
+
+
+
+
+
+
+
+ + + +
+
Memory
+
+
+ +
+
+ {{ memory.used }} used + {{ memory.total }} total +
+
-
- Current load - Normal +
+
+
+
+
+
+
+ + + + +
+
Disk Space
+
+
+ +
+
+ {{ disk.free }} free + {{ disk.total }} total +
+
+
+
+
+
+
+
+
+
+ + + +
+
Storage
+
+
+

{{ app.storage_used }}

+
+
+
{{ app.buckets }}
+ Buckets +
+
+
+
{{ app.objects }}
+ Objects +
+
+
+
+
+
+ + {{ app.buckets }} +
+
+ + {{ app.objects }} +
+
+ + {{ app.versions }} +
+
+
-
-
-
-
-
Memory
-
- - - -
-
-

{{ memory.percent }}%

-
-
-
-
- {{ memory.used }} used - {{ memory.total }} total -
-
-
-
- -
-
-
-
-
Disk Space
-
- - - - -
-
-

{{ disk.percent }}%

-
-
-
-
- {{ disk.free }} free - {{ disk.total }} total -
-
-
-
- -
-
-
-
-
Storage
-
- - - -
-
-

{{ app.storage_used }}

-
-
-
{{ app.buckets }}
- Buckets -
-
-
-
{{ app.objects }}
- Objects -
-
-
-
-
-
- -
-
-
-
-
System Overview
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ResourceValueStatus
-
-
- - - - - -
- Total Disk Capacity -
- 		{{ disk.total }}Hardware
-
-
- - - - -
- Available Space -
- 		{{ disk.free }} - {% if disk.percent > 90 %} - - Critical - - {% elif disk.percent > 75 %} - - Low - - {% else %} - - Good - - {% endif %} -
-
-
- - - -
- MyFSIO Data -
- 		{{ app.storage_used }}Application
-
-
- - - -
- Total Objects -
- 		{{ app.objects }}Count
-
-
-
-
- -
- {% set has_issues = (cpu_percent > 80) or (memory.percent > 85) or (disk.percent > 90) %} -
-
-
- + {% set has_issues = (cpu_percent > 80) or (memory.percent > 85) or (disk.percent > 90) %} +
+
+
+ + {% if has_issues %} + + {% else %} + + {% endif %} + +
+ {% if has_issues %}Issues Detected{% else %}All Systems Normal{% endif %} + {% if has_issues %} - - + {% if cpu_percent > 80 %}CPU {{ cpu_percent }}%{% endif %} + {% if memory.percent > 85 %}{% if cpu_percent > 80 %} · {% endif %}Memory {{ memory.percent }}%{% endif %} + {% if disk.percent > 90 %}{% if cpu_percent > 80 or memory.percent > 85 %} · {% endif %}Disk {{ disk.percent }}%{% endif %} {% else %} - - + All resources within normal operating parameters {% endif %} - -
-
- - - {% if has_issues %} - - {% else %} - - {% endif %} - - v{{ app.version }}
-

System Health

-
- {% if has_issues %} -
    - {% if cpu_percent > 80 %}
  • CPU usage is high ({{ cpu_percent }}%)
    • {% endif %} - {% if memory.percent > 85 %}
  • Memory usage is high ({{ memory.percent }}%)
    • {% endif %} - {% if disk.percent > 90 %}
  • Disk space is critically low ({{ disk.percent }}% used)
    • {% endif %} -
- {% else %} -

All resources are within normal operating parameters.

- {% endif %} +
+
+
+
{{ app.uptime_days }}d
+ Uptime
-
-
-
{{ app.uptime_days }}d
- Uptime -
-
-
{{ app.buckets }}
- Active Buckets -
+
+ v{{ app.version }}
@@ -279,110 +219,138 @@
{% if operation_metrics_enabled %} -
-
-
-
-
API Operations
-
- Loading... - +
+
+
+
+
+

0

+ Requests
-
-
-
-
-

0

- Requests -
-
-
-
-

0%

- Success -
-
-
-
-

0

- Errors -
-
-
-
-

0ms

- Latency -
-
-
-
-

0 B

- Bytes In -
-
-
-
-

0 B

- Bytes Out -
+
+
+
+
+

0%

+ Success +
+
+
+
+
+
+

0

+ Errors +
+
+
+
+
+
+

0

+ Req/s +
+
+
+
+
+
+

0ms

+ Avg Latency +
+
+
+
+
+
+

0ms

+ P95 Latency +
+
+
+
+
+
+

0 B

+ Bytes In +
+
+
+
+
+
+

0 B

+ Bytes Out +
+
+
+
+ +
+
+
+
+
Requests by Method
+
+
-
-
-
-
Requests by Method
-
- -
-
-
-
-
-
Requests by Status
-
- -
-
+
+
+
+
+
+
Requests by Status
+
+
-
-
-
-
Requests by Endpoint
-
- -
-
+
+
+
+
+
+
Requests by Endpoint
+
+
-
-
-
-
S3 Error Codes
- API Only -
-
-
-
Code
-
Count
-
Distribution
-
-
-
-
- - - - -
No S3 API errors
-
-
-
+
+
+
+
+ +
+
+
+
+
Latency by Endpoint
+
+ +
+
+
+
+
+
+
+
+
S3 Error Codes
+ API Only +
+
+
+
Code
+
Count
+
Distribution
+
+
+
+ + + + +
No S3 API errors
@@ -390,173 +358,304 @@
+ +
+
+
Operation Trends
+
+ Loading... + +
+
+
+
+
+
Request Rate (req/s)
+
+
+
+
Throughput (bytes/s)
+
+
+
+
+
+
Average Latency (ms)
+
+
+
+
Error Rate (%)
+
+
+
+

Loading operation history...

+
+
{% endif %} {% if metrics_history_enabled %} -
-
-
-
-
Metrics History
-
- - -
-
-
-
-
-
CPU Usage
- -
-
-
Memory Usage
- -
-
-
Disk Usage
- -
-
-

Loading history data...

-
+
+
+
+ + +
+ Loading... +
+
+
+
CPU Usage
+
+
+
+
+
+
Memory Usage
+
+
+
+
+
+
Disk Usage
+
{% endif %} + +
{% endblock %} {% block extra_scripts %} -{% if metrics_history_enabled or operation_metrics_enabled %} -{% endif %}