Optimize CPU usage via caching and reducing ThreadPoolExecutor workers to prevent CPU saturation

2026-02-02 13:30:06 +08:00
parent 45d21cce21
commit a779b002d7
5 changed files with 66 additions and 7 deletions
--- a/app/storage.py
+++ b/app/storage.py
@@ -1469,7 +1469,8 @@ class ObjectStorage:
            
            if meta_files:
                meta_cache = {}
-                with ThreadPoolExecutor(max_workers=min(64, len(meta_files))) as executor:
+                max_workers = min((os.cpu_count() or 4) * 2, len(meta_files), 16)
+                with ThreadPoolExecutor(max_workers=max_workers) as executor:
                    for key, etag in executor.map(read_meta_file, meta_files):
                        if etag:
                            meta_cache[key] = etag