fix: corrige 6 bugs mineurs (BUG-035 a BUG-040)
This commit is contained in:
+74
-6
@@ -481,12 +481,18 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
|
||||
|
||||
# PDF handling — special path (binary, uses pdf_reader)
|
||||
tags: list[str] = []
|
||||
pdf_text_pending = False
|
||||
if ext == ".pdf":
|
||||
from backend.pdf_reader import extract_pdf_metadata, extract_pdf_text
|
||||
raw = extract_pdf_text(fpath, max_chars=100000)
|
||||
from backend.pdf_reader import extract_pdf_metadata
|
||||
# BUG-040: only the (cheap) metadata is read during the
|
||||
# scan. Full-text extraction is deferred to a background
|
||||
# pass (``enrich_pdf_texts``) so a vault with many/large
|
||||
# PDFs no longer blocks startup and index rebuilds.
|
||||
pdf_meta = extract_pdf_metadata(fpath)
|
||||
title = pdf_meta.get("title") or fpath.stem.replace("-", " ").replace("_", " ")
|
||||
content_preview = raw[:200].strip()
|
||||
raw = ""
|
||||
content_preview = ""
|
||||
pdf_text_pending = True
|
||||
elif ext == ".excalidraw" or fpath.name.lower().endswith(".excalidraw.md"):
|
||||
raw = fpath.read_text(encoding="utf-8", errors="replace")
|
||||
raw = extract_excalidraw_indexable(raw)
|
||||
@@ -510,7 +516,7 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
|
||||
title, post.content
|
||||
)
|
||||
|
||||
files.append({
|
||||
file_info = {
|
||||
"path": str(relative).replace("\\", "/"),
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
@@ -519,7 +525,10 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
|
||||
"size": stat.st_size,
|
||||
"modified": modified,
|
||||
"extension": ext,
|
||||
})
|
||||
}
|
||||
if pdf_text_pending:
|
||||
file_info["pdf_text_pending"] = True
|
||||
files.append(file_info)
|
||||
|
||||
for tag in tags:
|
||||
tag_counts[tag] = tag_counts.get(tag, 0) + 1
|
||||
@@ -535,6 +544,60 @@ def _scan_vault(vault_name: str, vault_path: str, vault_cfg: dict[str, Any] | No
|
||||
return {"files": files, "tags": tag_counts, "path": vault_path, "paths": paths, "config": {}}
|
||||
|
||||
|
||||
async def enrich_pdf_texts(vault_name: str | None = None) -> int:
|
||||
"""Extract text from PDFs whose extraction was deferred during the scan (BUG-040).
|
||||
|
||||
``_scan_vault`` only reads PDF metadata so a vault with many or large PDFs
|
||||
starts serving immediately. This coroutine runs *after* the index (and the
|
||||
inverted index) is ready, extracts the missing text off the event loop and
|
||||
updates the in-memory entry plus the incremental index hooks.
|
||||
|
||||
Args:
|
||||
vault_name: Restrict the pass to a single vault; ``None`` covers every
|
||||
indexed vault.
|
||||
|
||||
Returns:
|
||||
Number of deferred PDFs whose text extraction was attempted.
|
||||
"""
|
||||
from backend.pdf_reader import extract_pdf_text
|
||||
|
||||
pending: list[tuple[str, dict[str, Any], Path]] = []
|
||||
with _index_lock:
|
||||
for name, vault_data in index.items():
|
||||
if vault_name is not None and name != vault_name:
|
||||
continue
|
||||
vault_root = Path(vault_data.get("path", ""))
|
||||
for file_info in vault_data.get("files", []):
|
||||
if file_info.get("pdf_text_pending"):
|
||||
pending.append((name, file_info, vault_root / file_info["path"]))
|
||||
|
||||
if not pending:
|
||||
return 0
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
enriched = 0
|
||||
for name, file_info, file_path in pending:
|
||||
try:
|
||||
raw = await loop.run_in_executor(None, extract_pdf_text, file_path, 100000)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.warning("PDF enrichment failed for %s: %s", file_path, exc)
|
||||
raw = ""
|
||||
file_info["content"] = raw[:SEARCH_CONTENT_LIMIT]
|
||||
file_info["content_preview"] = raw[:200].strip()
|
||||
file_info.pop("pdf_text_pending", None)
|
||||
enriched += 1
|
||||
if _on_index_change:
|
||||
try:
|
||||
_on_index_change("add", name, file_info["path"], file_info)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.warning(
|
||||
"Index hook failed after PDF enrichment for %s: %s", file_path, exc
|
||||
)
|
||||
|
||||
logger.info("PDF enrichment: extracted text for %d deferred PDF(s)", enriched)
|
||||
return enriched
|
||||
|
||||
|
||||
async def build_index(progress_callback=None) -> None:
|
||||
"""Build the full in-memory index for all configured vaults.
|
||||
|
||||
@@ -632,6 +695,8 @@ async def reload_index() -> dict[str, Any]:
|
||||
Dict mapping vault names to their file/tag counts.
|
||||
"""
|
||||
await build_index()
|
||||
# BUG-040: complete the deferred PDF extraction for the rebuilt index.
|
||||
await enrich_pdf_texts()
|
||||
stats = {}
|
||||
for name, data in index.items():
|
||||
stats[name] = {"file_count": len(data["files"]), "tag_count": len(data["tags"])}
|
||||
@@ -695,7 +760,10 @@ async def reload_single_vault(vault_name: str) -> dict[str, Any]:
|
||||
# Rebuild attachment index for this vault only
|
||||
from backend.attachment_indexer import build_attachment_index
|
||||
await build_attachment_index({vault_name: config})
|
||||
|
||||
|
||||
# BUG-040: complete the deferred PDF extraction for this vault.
|
||||
await enrich_pdf_texts(vault_name)
|
||||
|
||||
stats = {"file_count": len(vault_data["files"]), "tag_count": len(vault_data["tags"])}
|
||||
logger.info(f"Vault '{vault_name}' reindexed: {stats['file_count']} files, {stats['tag_count']} tags")
|
||||
return stats
|
||||
|
||||
Reference in New Issue
Block a user