feat: optimisation globale des performances #86 (scan differentiel, excalidraw differe, garde-fou replace; inverted index/PDF lazy/caps regex deja livres via BUG-033/040/025)

backend/indexer.py

@@ -397,31 +397,52 @@ def parse_markdown_file(raw: str) -> frontmatter.Post:
         return frontmatter.Post(content)
 
 
-def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | None = None) -> dict[str, Any]:
+def _scan_vault(
+    vault_name: str,
+    vault_path: str,
+    vault_cfg: dict[str, Any] | None = None,
+    previous_files: dict[str, dict[str, Any]] | None = None,
+) -> dict[str, Any]:
     """Synchronously scan a single vault directory and build file index.
 
     Walks the vault tree, reads supported files, extracts metadata
     (tags, title, content preview) and stores a capped content snapshot
     for in-memory full-text search.
-    
+
     All files and directories are indexed, including hidden files (starting with '.').
 
+    Differential scan (#86): when ``previous_files`` maps a relative path to
+    its previous ``file_info`` dict, entries whose ``size`` and ``modified``
+    timestamp are unchanged are reused verbatim (no disk read, no re-parse).
+    Only the cheap ``os.walk`` + ``stat`` runs on every pass; heavy content
+    extraction (PDF metadata excepted — always cheap) is skipped for
+    unchanged files. This replaces the full ``rglob`` re-read on rebuilds.
+
+    Excalidraw diagrams (#86, like PDFs since BUG-040) are deferred: the scan
+    only records the title and sets ``excalidraw_text_pending``; the expensive
+    JSON/lz-string text extraction runs in ``enrich_pdf_texts()`` after the
+    index is queryable.
+
     Args:
         vault_name: Display name of the vault.
         vault_path: Absolute filesystem path to the vault root.
         vault_cfg: Optional vault configuration dict (unused for indexing, kept for compatibility).
+        previous_files: Optional ``{relative_path: file_info}`` snapshot from a
+            previous scan used for differential reuse.
 
     Returns:
-        Dict with keys ``files`` (list), ``tags`` (counter dict), ``path`` (str), ``paths`` (list).
+        Dict with keys ``files`` (list), ``tags`` (counter dict), ``path`` (str),
+        ``paths`` (list) and ``reused`` (int, differential hits).
     """
     vault_root = Path(vault_path)
     files: list[dict[str, Any]] = []
     tag_counts: dict[str, int] = {}
     paths: list[dict[str, str]] = []
+    reused = 0
 
     if not vault_root.exists():
         logger.warning(f"Vault path does not exist: {vault_path}")
-        return {"files": [], "tags": {}, "path": vault_path, "paths": []}
+        return {"files": [], "tags": {}, "path": vault_path, "paths": [], "reused": 0}
 
     root_resolved = vault_root.resolve(strict=False)
 
@@ -479,9 +500,37 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
                 stat = fpath.stat()
                 modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat()
 
+                # #86 differential scan: reuse the previous entry when neither
+                # size nor mtime changed — skips the disk read + parse below.
+                if previous_files:
+                    prev = previous_files.get(rel_path_str)
+                    if (
+                        prev is not None
+                        and prev.get("size") == stat.st_size
+                        and prev.get("modified") == modified
+                    ):
+                        file_info = {**prev, "tags": list(prev.get("tags", []))}
+                        files.append(file_info)
+                        for tag in file_info.get("tags", []):
+                            tag_counts[tag] = tag_counts.get(tag, 0) + 1
+                        reused += 1
+                        # The global backlink index is rebuilt on every scan,
+                        # so re-register this file's wikilinks from its
+                        # (cached) content instead of re-reading the disk.
+                        if file_info.get("extension") == ".md" and file_info.get("content"):
+                            try:
+                                _extract_wikilinks_for_backlinks(
+                                    vault_name, file_info["path"],
+                                    file_info.get("title", ""), file_info["content"],
+                                )
+                            except Exception:
+                                pass
+                        continue
+
                 # PDF handling — special path (binary, uses pdf_reader)
                 tags: list[str] = []
                 pdf_text_pending = False
+                excalidraw_text_pending = False
                 if ext == ".pdf":
                     from backend.pdf_reader import extract_pdf_metadata
                     # BUG-040: only the (cheap) metadata is read during the
@@ -494,10 +543,13 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
                     content_preview = ""
                     pdf_text_pending = True
                 elif ext == ".excalidraw" or fpath.name.lower().endswith(".excalidraw.md"):
-                    raw = fpath.read_text(encoding="utf-8", errors="replace")
-                    raw = extract_excalidraw_indexable(raw)
+                    # #86: defer the expensive JSON/lz-string text extraction
+                    # (read + decompress + element walk) to ``enrich_pdf_texts``
+                    # so the scan stays cheap; title comes from the filename.
+                    raw = ""
                     title = fpath.stem.replace(".excalidraw", "").replace("-", " ").replace("_", " ")
-                    content_preview = raw[:200].strip()
+                    content_preview = ""
+                    excalidraw_text_pending = True
                 else:
                     raw = fpath.read_text(encoding="utf-8", errors="replace")
                     title = fpath.stem.replace("-", " ").replace("_", " ")
@@ -528,6 +580,8 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
                 }
                 if pdf_text_pending:
                     file_info["pdf_text_pending"] = True
+                if excalidraw_text_pending:
+                    file_info["excalidraw_text_pending"] = True
                 files.append(file_info)
 
                 for tag in tags:
@@ -540,28 +594,49 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
                 logger.error(f"Error indexing {fpath}: {e}")
                 continue
 
-    logger.info(f"Vault '{vault_name}': indexed {len(files)} files, {len(paths)} paths, {len(tag_counts)} unique tags")
-    return {"files": files, "tags": tag_counts, "path": vault_path, "paths": paths, "config": {}}
+    logger.info(
+        f"Vault '{vault_name}': indexed {len(files)} files "
+        f"({reused} reused), {len(paths)} paths, {len(tag_counts)} unique tags"
+    )
+    return {"files": files, "tags": tag_counts, "path": vault_path, "paths": paths, "config": {}, "reused": reused}
+
+
+def _read_excalidraw_indexable_text(file_path: Path) -> str:
+    """Read an excalidraw file and return its indexable text (blocking helper).
+
+    Runs inside an executor via ``enrich_pdf_texts`` so the lz-string
+    decompression of large diagrams never blocks the event loop.
+    """
+    try:
+        raw = file_path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return ""
+    try:
+        return extract_excalidraw_indexable(raw)
+    except Exception:  # pragma: no cover - defensive
+        return ""
 
 
 async def enrich_pdf_texts(vault_name: str | None = None) -> int:
-    """Extract text from PDFs whose extraction was deferred during the scan (BUG-040).
+    """Extract text deferred during the scan: PDFs (BUG-040) + excalidraw (#86).
 
-    ``_scan_vault`` only reads PDF metadata so a vault with many or large PDFs
-    starts serving immediately. This coroutine runs *after* the index (and the
-    inverted index) is ready, extracts the missing text off the event loop and
-    updates the in-memory entry plus the incremental index hooks.
+    ``_scan_vault`` only reads PDF metadata and excalidraw filenames so a vault
+    with many or large heavy files starts serving immediately. This coroutine
+    runs *after* the index (and the inverted index) is ready, extracts the
+    missing text off the event loop and updates the in-memory entry plus the
+    incremental index hooks.
 
     Args:
         vault_name: Restrict the pass to a single vault; ``None`` covers every
             indexed vault.
 
     Returns:
-        Number of deferred PDFs whose text extraction was attempted.
+        Number of deferred files (PDF + excalidraw) whose text extraction was
+        attempted.
     """
     from backend.pdf_reader import extract_pdf_text
 
-    pending: list[tuple[str, dict[str, Any], Path]] = []
+    pending: list[tuple[str, dict[str, Any], Path, str]] = []
     with _index_lock:
         for name, vault_data in index.items():
             if vault_name is not None and name != vault_name:
@@ -569,32 +644,38 @@ async def enrich_pdf_texts(vault_name: str | None = None) -> int:
             vault_root = Path(vault_data.get("path", ""))
             for file_info in vault_data.get("files", []):
                 if file_info.get("pdf_text_pending"):
-                    pending.append((name, file_info, vault_root / file_info["path"]))
+                    pending.append((name, file_info, vault_root / file_info["path"], "pdf"))
+                elif file_info.get("excalidraw_text_pending"):
+                    pending.append((name, file_info, vault_root / file_info["path"], "excalidraw"))
 
     if not pending:
         return 0
 
     loop = asyncio.get_running_loop()
     enriched = 0
-    for name, file_info, file_path in pending:
+    for name, file_info, file_path, kind in pending:
         try:
-            raw = await loop.run_in_executor(None, extract_pdf_text, file_path, 100000)
+            if kind == "pdf":
+                raw = await loop.run_in_executor(None, extract_pdf_text, file_path, 100000)
+            else:
+                raw = await loop.run_in_executor(None, _read_excalidraw_indexable_text, file_path)
         except Exception as exc:  # pragma: no cover - defensive
-            logger.warning("PDF enrichment failed for %s: %s", file_path, exc)
+            logger.warning("Deferred text enrichment failed for %s: %s", file_path, exc)
             raw = ""
         file_info["content"] = raw[:SEARCH_CONTENT_LIMIT]
         file_info["content_preview"] = raw[:200].strip()
         file_info.pop("pdf_text_pending", None)
+        file_info.pop("excalidraw_text_pending", None)
         enriched += 1
         if _on_index_change:
             try:
                 _on_index_change("add", name, file_info["path"], file_info)
             except Exception as exc:  # pragma: no cover - defensive
                 logger.warning(
-                    "Index hook failed after PDF enrichment for %s: %s", file_path, exc
+                    "Index hook failed after deferred enrichment for %s: %s", file_path, exc
                 )
 
-    logger.info("PDF enrichment: extracted text for %d deferred PDF(s)", enriched)
+    logger.info("Deferred text enrichment: extracted text for %d file(s)", enriched)
     return enriched
 
 
@@ -603,16 +684,24 @@ async def build_index(progress_callback=None) -> None:
 
     Runs vault scans concurrently, inserting them incrementally into the global index.
     Notifies progress via the provided callback.
+
+    #86 differential rebuild: the previous per-vault ``{path: file_info}``
+    snapshots are captured before the clear and handed to ``_scan_vault`` so
+    unchanged files (same size + mtime) are reused without disk re-reads.
     """
     global index, vault_config
     vault_config.clear()
     vault_config.update(load_vault_config())
-    
+
     # Note: vault_settings are now only used for UI display preferences (hideHiddenFiles)
     # Indexing always includes all files regardless of settings
-    
+
     global _index_generation
     with _index_lock:
+        previous_snapshot: dict[str, dict[str, dict[str, Any]]] = {
+            name: {f["path"]: f for f in vdata.get("files", [])}
+            for name, vdata in index.items()
+        }
         index.clear()
         _file_lookup.clear()
         path_index.clear()
@@ -631,8 +720,13 @@ async def build_index(progress_callback=None) -> None:
     loop = asyncio.get_event_loop()
 
     async def _process_vault(name: str, config: dict[str, Any]):
+        import functools
+
         vault_path = config["path"]
-        vault_data = await loop.run_in_executor(None, _scan_vault, name, vault_path, config)
+        scan = functools.partial(
+            _scan_vault, name, vault_path, config, previous_snapshot.get(name)
+        )
+        vault_data = await loop.run_in_executor(None, scan)
         vault_data["config"] = config
 
         # Build lookup entries for the new vault
@@ -695,7 +789,7 @@ async def reload_index() -> dict[str, Any]:
         Dict mapping vault names to their file/tag counts.
     """
     await build_index()
-    # BUG-040: complete the deferred PDF extraction for the rebuilt index.
+    # BUG-040/#86: complete the deferred PDF + excalidraw extraction.
     await enrich_pdf_texts()
     stats = {}
     for name, data in index.items():
@@ -724,14 +818,22 @@ async def reload_single_vault(vault_name: str) -> dict[str, Any]:
         raise ValueError(f"Vault '{vault_name}' not found in configuration")
     
     config = vault_config[vault_name]
-    
+
+    # #86 differential rescan: snapshot this vault's entries before removal so
+    # unchanged files are reused without disk re-reads.
+    with _index_lock:
+        _previous = {f["path"]: f for f in index.get(vault_name, {}).get("files", [])}
+
     # Remove old vault data from index structures
     await remove_vault_from_index(vault_name)
-    
+
     # Re-add the vault with updated configuration
+    import functools
+
     vault_path = config["path"]
     loop = asyncio.get_event_loop()
-    vault_data = await loop.run_in_executor(None, _scan_vault, vault_name, vault_path, config)
+    scan = functools.partial(_scan_vault, vault_name, vault_path, config, _previous)
+    vault_data = await loop.run_in_executor(None, scan)
     vault_data["config"] = config
     
     # Build lookup entries for the vault
@@ -761,7 +863,7 @@ async def reload_single_vault(vault_name: str) -> dict[str, Any]:
     from backend.attachment_indexer import build_attachment_index
     await build_attachment_index({vault_name: config})
 
-    # BUG-040: complete the deferred PDF extraction for this vault.
+    # BUG-040/#86: complete the deferred PDF + excalidraw extraction.
     await enrich_pdf_texts(vault_name)
 
     stats = {"file_count": len(vault_data["files"]), "tag_count": len(vault_data["tags"])}

backend/services/mutations.py

@@ -26,6 +26,11 @@ from backend.services.vaults import get_vault_root
 
 logger = logging.getLogger("obsigate.services.mutations")
 
+# #86: per-file size cap for find/replace passes (CPU guard — complements the
+# BUG-025 regex caps). Files larger than this are skipped instead of being
+# read fully into memory and scanned with a user-supplied pattern.
+MAX_REPLACE_FILE_BYTES = 5_000_000
+
 # Skeleton injected into empty ``.excalidraw`` files (mirrors the route logic).
 _EXCALIDRAW_SKELETON = (
     '{"type":"excalidraw","version":2,"elements":[],'
@@ -509,6 +514,16 @@ def replace_in_files(
             continue
         if not file_path.exists() or not file_path.is_file():
             continue
+        # #86 CPU guard: skip files too large to scan safely in one pass.
+        try:
+            if file_path.stat().st_size > MAX_REPLACE_FILE_BYTES:
+                logger.warning(
+                    "replace_in_files: skipping oversized file %s/%s (%d bytes)",
+                    result_vault, result["path"], file_path.stat().st_size,
+                )
+                continue
+        except OSError:
+            continue
         try:
             original = file_path.read_text(encoding="utf-8", errors="replace")
         except OSError: