Make integrity scan async with progress indicator in UI

app/admin_api.py

@@ -961,12 +961,14 @@ def integrity_run_now():
     payload = request.get_json(silent=True) or {}
     override_dry_run = payload.get("dry_run")
     override_auto_heal = payload.get("auto_heal")
-    result = checker.run_now(
+    started = checker.run_async(
         auto_heal=override_auto_heal if override_auto_heal is not None else None,
         dry_run=override_dry_run if override_dry_run is not None else None,
     )
     logger.info("Integrity manual run by %s", principal.access_key)
-    return jsonify(result.to_dict())
+    if not started:
+        return _json_error("Conflict", "A scan is already in progress", 409)
+    return jsonify({"status": "started"})
 
 
 @admin_api_bp.route("/integrity/history", methods=["GET"])

app/integrity.py

@@ -189,6 +189,8 @@ class IntegrityChecker:
         self._timer: Optional[threading.Timer] = None
         self._shutdown = False
         self._lock = threading.Lock()
+        self._scanning = False
+        self._scan_start_time: Optional[float] = None
         self.history_store = IntegrityHistoryStore(storage_root, max_records=max_history)
 
     def start(self) -> None:
@@ -229,10 +231,17 @@ class IntegrityChecker:
             self._schedule_next()
 
     def run_now(self, auto_heal: Optional[bool] = None, dry_run: Optional[bool] = None) -> IntegrityResult:
+        if not self._lock.acquire(blocking=False):
+            raise RuntimeError("Integrity scan is already in progress")
+
+        try:
+            self._scanning = True
+            self._scan_start_time = time.time()
+
             effective_auto_heal = auto_heal if auto_heal is not None else self.auto_heal
             effective_dry_run = dry_run if dry_run is not None else self.dry_run
 
-        start = time.time()
+            start = self._scan_start_time
             result = IntegrityResult()
 
             bucket_names = self._list_bucket_names()
@@ -275,6 +284,17 @@ class IntegrityChecker:
             self.history_store.add(record)
 
             return result
+        finally:
+            self._scanning = False
+            self._scan_start_time = None
+            self._lock.release()
+
+    def run_async(self, auto_heal: Optional[bool] = None, dry_run: Optional[bool] = None) -> bool:
+        if self._scanning:
+            return False
+        t = threading.Thread(target=self.run_now, args=(auto_heal, dry_run), daemon=True)
+        t.start()
+        return True
 
     def _system_path(self) -> Path:
         return self.storage_root / self.SYSTEM_ROOT
@@ -728,11 +748,15 @@ class IntegrityChecker:
         return [r.to_dict() for r in records]
 
     def get_status(self) -> dict:
-        return {
+        status: Dict[str, Any] = {
             "enabled": not self._shutdown or self._timer is not None,
             "running": self._timer is not None and not self._shutdown,
+            "scanning": self._scanning,
             "interval_hours": self.interval_seconds / 3600.0,
             "batch_size": self.batch_size,
             "auto_heal": self.auto_heal,
             "dry_run": self.dry_run,
         }
+        if self._scanning and self._scan_start_time is not None:
+            status["scan_elapsed_seconds"] = round(time.time() - self._scan_start_time, 1)
+        return status

app/ui.py

@@ -4202,11 +4202,46 @@ def system_integrity_run():
         return jsonify({"error": "Integrity checker is not enabled"}), 400
 
     payload = request.get_json(silent=True) or {}
-    result = checker.run_now(
+    started = checker.run_async(
         auto_heal=payload.get("auto_heal"),
         dry_run=payload.get("dry_run"),
     )
-    return jsonify(result.to_dict())
+    if not started:
+        return jsonify({"error": "A scan is already in progress"}), 409
+    return jsonify({"status": "started"})
+
+
+@ui_bp.get("/system/integrity/status")
+def system_integrity_status():
+    principal = _current_principal()
+    try:
+        _iam().authorize(principal, None, "iam:*")
+    except IamError:
+        return jsonify({"error": "Access denied"}), 403
+
+    checker = current_app.extensions.get("integrity")
+    if not checker:
+        return jsonify({"error": "Integrity checker is not enabled"}), 400
+
+    return jsonify(checker.get_status())
+
+
+@ui_bp.get("/system/integrity/history")
+def system_integrity_history():
+    principal = _current_principal()
+    try:
+        _iam().authorize(principal, None, "iam:*")
+    except IamError:
+        return jsonify({"error": "Access denied"}), 403
+
+    checker = current_app.extensions.get("integrity")
+    if not checker:
+        return jsonify({"executions": []})
+
+    limit = min(int(request.args.get("limit", 10)), 200)
+    offset = int(request.args.get("offset", 0))
+    records = checker.get_history(limit=limit, offset=offset)
+    return jsonify({"executions": records})
 
 
 @ui_bp.app_errorhandler(404)

app/version.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-APP_VERSION = "0.4.0"
+APP_VERSION = "0.4.1"
 
 
 def get_version() -> str:

templates/system.html

@@ -233,21 +233,28 @@
       <div class="card-body px-4 pb-4">
         {% if integrity_status.enabled %}
         <div class="d-flex gap-2 flex-wrap mb-3">
-          <button class="btn btn-primary btn-sm d-inline-flex align-items-center" id="integrityRunBtn" onclick="runIntegrity(false, false)">
+          <button class="btn btn-primary btn-sm d-inline-flex align-items-center" id="integrityRunBtn" onclick="runIntegrity(false, false)" {% if integrity_status.scanning %}disabled{% endif %}>
             <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1 flex-shrink-0" viewBox="0 0 16 16">
               <path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>
               <path d="M8 4.466V.534a.25.25 0 0 1 .41-.192l2.36 1.966c.12.1.12.284 0 .384L8.41 4.658A.25.25 0 0 1 8 4.466z"/>
             </svg>
             Scan Now
           </button>
-          <button class="btn btn-outline-warning btn-sm" id="integrityHealBtn" onclick="runIntegrity(false, true)">
+          <button class="btn btn-outline-warning btn-sm" id="integrityHealBtn" onclick="runIntegrity(false, true)" {% if integrity_status.scanning %}disabled{% endif %}>
             Scan &amp; Heal
           </button>
-          <button class="btn btn-outline-secondary btn-sm" id="integrityDryRunBtn" onclick="runIntegrity(true, false)">
+          <button class="btn btn-outline-secondary btn-sm" id="integrityDryRunBtn" onclick="runIntegrity(true, false)" {% if integrity_status.scanning %}disabled{% endif %}>
             Dry Run
           </button>
         </div>
 
+        <div id="integrityScanningBanner" class="mb-3 {% if not integrity_status.scanning %}d-none{% endif %}">
+          <div class="alert alert-info mb-0 small d-flex align-items-center gap-2">
+            <div class="spinner-border spinner-border-sm text-info" role="status"></div>
+            <span>Scan in progress<span id="integrityScanElapsed"></span></span>
+          </div>
+        </div>
+
         <div id="integrityResult" class="mb-3 d-none">
           <div class="alert mb-0 small" id="integrityResultAlert">
             <div class="d-flex justify-content-between align-items-start">
@@ -431,32 +438,35 @@
     });
   };
 
-  window.runIntegrity = function (dryRun, autoHeal) {
+  var _integrityPollTimer = null;
-    var activeBtn = dryRun ? 'integrityDryRunBtn' : (autoHeal ? 'integrityHealBtn' : 'integrityRunBtn');
+  var _integrityLastMode = {dryRun: false, autoHeal: false};
-    ['integrityRunBtn', 'integrityHealBtn', 'integrityDryRunBtn'].forEach(function (id) {
-      setLoading(id, true, id !== activeBtn);
-    });
 
-    fetch('{{ url_for("ui.system_integrity_run") }}', {
+  function _integritySetScanning(scanning) {
-      method: 'POST',
+    var banner = document.getElementById('integrityScanningBanner');
-      headers: {'Content-Type': 'application/json', 'X-CSRFToken': csrfToken},
+    var btns = ['integrityRunBtn', 'integrityHealBtn', 'integrityDryRunBtn'];
-      body: JSON.stringify({dry_run: dryRun, auto_heal: autoHeal})
+    if (scanning) {
-    })
+      banner.classList.remove('d-none');
-    .then(function (r) { return r.json(); })
+      btns.forEach(function (id) {
-    .then(function (data) {
+        var el = document.getElementById(id);
+        if (el) el.disabled = true;
+      });
+    } else {
+      banner.classList.add('d-none');
+      document.getElementById('integrityScanElapsed').textContent = '';
+      btns.forEach(function (id) {
+        var el = document.getElementById(id);
+        if (el) el.disabled = false;
+      });
+    }
+  }
+
+  function _integrityShowResult(data, dryRun, autoHeal) {
     var container = document.getElementById('integrityResult');
     var alert = document.getElementById('integrityResultAlert');
     var title = document.getElementById('integrityResultTitle');
     var body = document.getElementById('integrityResultBody');
     container.classList.remove('d-none');
 
-      if (data.error) {
-        alert.className = 'alert alert-danger mb-0 small';
-        title.textContent = 'Error';
-        body.textContent = data.error;
-        return;
-      }
-
     var totalIssues = (data.corrupted_objects || 0) + (data.orphaned_objects || 0) +
                        (data.phantom_metadata || 0) + (data.stale_versions || 0) +
                        (data.etag_cache_inconsistencies || 0) + (data.legacy_metadata_drifts || 0);
@@ -481,8 +491,66 @@
     if (data.errors && data.errors.length > 0) lines.push('Errors: ' + data.errors.join(', '));
 
     body.innerHTML = lines.join('<br>');
+  }
+
+  function _integrityPoll() {
+    fetch('{{ url_for("ui.system_integrity_status") }}', {
+      headers: {'X-CSRFToken': csrfToken}
+    })
+    .then(function (r) { return r.json(); })
+    .then(function (status) {
+      if (status.scanning) {
+        var elapsed = status.scan_elapsed_seconds || 0;
+        document.getElementById('integrityScanElapsed').textContent = ' (' + elapsed.toFixed(0) + 's)';
+        _integrityPollTimer = setTimeout(_integrityPoll, 2000);
+      } else {
+        _integritySetScanning(false);
+        fetch('{{ url_for("ui.system_integrity_history") }}?limit=1', {
+          headers: {'X-CSRFToken': csrfToken}
+        })
+        .then(function (r) { return r.json(); })
+        .then(function (hist) {
+          if (hist.executions && hist.executions.length > 0) {
+            var latest = hist.executions[0];
+            _integrityShowResult(latest.result, latest.dry_run, latest.auto_heal);
+          }
+        })
+        .catch(function () {});
+      }
+    })
+    .catch(function () {
+      _integrityPollTimer = setTimeout(_integrityPoll, 3000);
+    });
+  }
+
+  window.runIntegrity = function (dryRun, autoHeal) {
+    _integrityLastMode = {dryRun: dryRun, autoHeal: autoHeal};
+    document.getElementById('integrityResult').classList.add('d-none');
+    _integritySetScanning(true);
+
+    fetch('{{ url_for("ui.system_integrity_run") }}', {
+      method: 'POST',
+      headers: {'Content-Type': 'application/json', 'X-CSRFToken': csrfToken},
+      body: JSON.stringify({dry_run: dryRun, auto_heal: autoHeal})
+    })
+    .then(function (r) { return r.json(); })
+    .then(function (data) {
+      if (data.error) {
+        _integritySetScanning(false);
+        var container = document.getElementById('integrityResult');
+        var alert = document.getElementById('integrityResultAlert');
+        var title = document.getElementById('integrityResultTitle');
+        var body = document.getElementById('integrityResultBody');
+        container.classList.remove('d-none');
+        alert.className = 'alert alert-danger mb-0 small';
+        title.textContent = 'Error';
+        body.textContent = data.error;
+        return;
+      }
+      _integrityPollTimer = setTimeout(_integrityPoll, 2000);
     })
     .catch(function (err) {
+      _integritySetScanning(false);
       var container = document.getElementById('integrityResult');
       var alert = document.getElementById('integrityResultAlert');
       var title = document.getElementById('integrityResultTitle');
@@ -491,13 +559,13 @@
       alert.className = 'alert alert-danger mb-0 small';
       title.textContent = 'Error';
       body.textContent = err.message;
-    })
-    .finally(function () {
-      setLoading('integrityRunBtn', false);
-      setLoading('integrityHealBtn', false);
-      setLoading('integrityDryRunBtn', false);
     });
   };
+
+  {% if integrity_status.scanning %}
+  _integritySetScanning(true);
+  _integrityPollTimer = setTimeout(_integrityPoll, 2000);
+  {% endif %}
 })();
 </script>
 {% endblock %}

tests/test_integrity.py

@@ -2,6 +2,7 @@ import hashlib
 import json
 import os
 import sys
+import time
 from pathlib import Path
 
 import pytest
@@ -11,6 +12,17 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 from app.integrity import IntegrityChecker, IntegrityResult
 
 
+def _wait_scan_done(client, headers, timeout=10):
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        resp = client.get("/admin/integrity/status", headers=headers)
+        data = resp.get_json()
+        if not data.get("scanning"):
+            return
+        time.sleep(0.1)
+    raise TimeoutError("scan did not complete")
+
+
 def _md5(data: bytes) -> str:
     return hashlib.md5(data).hexdigest()
 
@@ -413,8 +425,13 @@ class TestAdminAPI:
         resp = client.post("/admin/integrity/run", headers=AUTH_HEADERS, json={})
         assert resp.status_code == 200
         data = resp.get_json()
-        assert "corrupted_objects" in data
+        assert data["status"] == "started"
-        assert "objects_scanned" in data
+        _wait_scan_done(client, AUTH_HEADERS)
+        resp = client.get("/admin/integrity/history?limit=1", headers=AUTH_HEADERS)
+        hist = resp.get_json()
+        assert len(hist["executions"]) >= 1
+        assert "corrupted_objects" in hist["executions"][0]["result"]
+        assert "objects_scanned" in hist["executions"][0]["result"]
 
     def test_run_with_overrides(self, integrity_app):
         client = integrity_app.test_client()
@@ -424,10 +441,12 @@ class TestAdminAPI:
             json={"dry_run": True, "auto_heal": True},
         )
         assert resp.status_code == 200
+        _wait_scan_done(client, AUTH_HEADERS)
 
     def test_history_endpoint(self, integrity_app):
         client = integrity_app.test_client()
         client.post("/admin/integrity/run", headers=AUTH_HEADERS, json={})
+        _wait_scan_done(client, AUTH_HEADERS)
         resp = client.get("/admin/integrity/history", headers=AUTH_HEADERS)
         assert resp.status_code == 200
         data = resp.get_json()