Projets/ObsiGate
1
1
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Switch inverted index from stale check to incremental updates

Browse Source 
Register a hook with the indexer so that file add/remove events
incrementally maintain the inverted index, removing the need for
periodic staleness checks and cooldowns. Rebuild the index once on
startup via init_inverted_index().
This commit is contained in:
bruno
2026-05-26 12:37:59 -04:00
parent 872a3e56dd
commit 775722f5d4
4 changed files with 460 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/search.py
+149 -27
View File
@@ -239,9 +239,9 @@ def _escape_html(text: str) -> str:
class InvertedIndex:
"""In-memory inverted index supporting TF-IDF scoring.
Built lazily from the global ``index`` dict whenever a search or
suggestion request detects that the underlying vault index has changed.
The class is designed to be a singleton — use ``get_inverted_index()``.
Built initially via ``rebuild()`` from the global index, then
maintained incrementally via ``add_document()`` / ``remove_document()``
hooks from the file watcher and API mutations.
Attributes:
word_index: ``{token: {doc_key: term_frequency}}``
@@ -249,7 +249,6 @@ class InvertedIndex:
tag_norm_map: ``{normalized_tag: original_tag}``
tag_prefix_index: ``{prefix: [original_tag, ...]}``
doc_count: Total number of indexed documents.
_source_id: Fingerprint of the source index to detect staleness.
"""
def __init__(self) -> None:
@@ -264,23 +263,7 @@ class InvertedIndex:
self.vault_docs: Dict[str, set] = defaultdict(set)
self.tag_docs: Dict[str, set] = defaultdict(set)
self._sorted_tokens: List[str] = []
self._source_generation: int = -1
self._last_rebuild: float = 0
self._rebuild_cooldown: float = 3.0 # seconds
def is_stale(self) -> bool:
"""Check if the inverted index needs rebuilding.
Uses a cooldown (3s) to prevent rapid rebuilds from file watcher
events. Staleness is only reported if the generation has changed
AND the cooldown has elapsed since the last rebuild.
"""
import time
if _indexer._index_generation == self._source_generation:
return False
if time.time() - self._last_rebuild < self._rebuild_cooldown:
return False
return True
self._ready: bool = False # True after initial build
def rebuild(self) -> None:
"""Rebuild inverted index from the global ``index`` dict.
@@ -288,9 +271,7 @@ class InvertedIndex:
Tokenizes titles and content of every file, computes term frequencies,
and builds auxiliary indexes for tag and title prefix suggestions.
"""
import time
self._last_rebuild = time.time()
logger.info("Rebuilding inverted index...")
logger.info("Building inverted index...")
self.word_index = defaultdict(dict)
self.title_index = defaultdict(list)
self.tag_norm_map = {}
@@ -351,7 +332,7 @@ class InvertedIndex:
self.tag_prefix_index[prefix].append(tag)
self._sorted_tokens = sorted(self.word_index.keys())
self._source_generation = _indexer._index_generation
self._ready = True
logger.info(
"Inverted index built: %d documents, %d unique tokens, %d tags",
self.doc_count,
@@ -359,6 +340,117 @@ class InvertedIndex:
len(self.tag_norm_map),
)
def add_document(self, vault_name: str, path: str, file_info: dict):
"""Add or update a single document incrementally."""
if not self._ready:
return
doc_key = f"{vault_name}::{path}"
old_file_info = self.doc_info.get(doc_key)
if old_file_info is not None:
self._remove_doc_internals(doc_key, vault_name, old_file_info, skip_sorted_cleanup=True)
else:
self.doc_count += 1
# Metadata
self.doc_info[doc_key] = file_info
self.doc_vault[doc_key] = vault_name
self.vault_docs[vault_name].add(doc_key)
# Tags
tags = file_info.get("tags", [])
for tag in tags:
self.tag_docs[tag.lower()].add(doc_key)
norm_tag = normalize_text(tag)
if norm_tag not in self.tag_norm_map:
self.tag_norm_map[norm_tag] = tag
for plen in range(MIN_PREFIX_LENGTH, len(norm_tag) + 1):
prefix = norm_tag[:plen]
if tag not in self.tag_prefix_index[prefix]:
self.tag_prefix_index[prefix].append(tag)
# Title tokens
title = file_info.get("title", "")
title_tokens = tokenize(title)
for token in set(title_tokens):
if token:
self.title_index[token].append(doc_key)
# Title norm map
norm_title = normalize_text(title)
if norm_title:
self.title_norm_map[norm_title].append({"vault": vault_name, "path": path, "title": title})
# Word index (content + title TF)
content = file_info.get("content", "")
full_text = title + " " + content
tokens = tokenize(full_text)
tf: Dict[str, int] = defaultdict(int)
for token in tokens:
if token:
tf[token] += 1
for token, freq in tf.items():
if not self.word_index.get(token):
bisect.insort(self._sorted_tokens, token)
self.word_index[token][doc_key] = freq
def remove_document(self, vault_name: str, path: str):
"""Remove a single document incrementally."""
if not self._ready:
return
doc_key = f"{vault_name}::{path}"
file_info = self.doc_info.get(doc_key)
if file_info is None:
return
self._remove_doc_internals(doc_key, vault_name, file_info, skip_sorted_cleanup=False)
self.doc_count -= 1
def _remove_doc_internals(self, doc_key: str, vault_name: str, file_info: dict, skip_sorted_cleanup: bool = False):
"""Remove one doc_key from all indexes without adjusting doc_count."""
# Metadata
self.doc_info.pop(doc_key, None)
self.doc_vault.pop(doc_key, None)
if vault_name in self.vault_docs:
self.vault_docs[vault_name].discard(doc_key)
# Tags (per-document, NOT the global tag_norm_map)
for tag in file_info.get("tags", []):
td = self.tag_docs.get(tag.lower())
if td:
td.discard(doc_key)
if not td:
del self.tag_docs[tag.lower()]
# Title tokens
title = file_info.get("title", "")
for token in set(tokenize(title)):
if not token:
continue
ti = self.title_index.get(token)
if ti:
try:
ti.remove(doc_key)
except ValueError:
pass
if not ti:
del self.title_index[token]
# Title norm map
norm_title = normalize_text(title)
if norm_title and norm_title in self.title_norm_map:
self.title_norm_map[norm_title] = [
e for e in self.title_norm_map[norm_title]
if not (e["vault"] == vault_name and e["path"] == file_info.get("path"))
]
if not self.title_norm_map[norm_title]:
del self.title_norm_map[norm_title]
# Word index
content = file_info.get("content", "")
full_text = title + " " + content
for token in set(tokenize(full_text)):
if not token:
continue
wi = self.word_index.get(token)
if wi:
wi.pop(doc_key, None)
if not wi:
del self.word_index[token]
if not skip_sorted_cleanup:
idx = bisect.bisect_left(self._sorted_tokens, token)
if idx < len(self._sorted_tokens) and self._sorted_tokens[idx] == token:
self._sorted_tokens.pop(idx)
def idf(self, term: str) -> float:
"""Inverse Document Frequency for a term.
@@ -424,9 +516,39 @@ class InvertedIndex:
_inverted_index = InvertedIndex()
def _on_index_change_hook(action: str, vault_name: str, path: str, file_info: dict):
"""Callback registered with indexer for incremental inverted index updates."""
inv = _inverted_index
try:
if action == 'add' and file_info:
inv.add_document(vault_name, path, file_info)
elif action == 'remove':
inv.remove_document(vault_name, path)
except Exception as e:
logger.warning(f"Inverted index incremental update failed ({action} {vault_name}/{path}): {e}")
# Register the hook with indexer (indexer is already imported at top of file)
_indexer.set_index_change_hook(_on_index_change_hook)
def init_inverted_index():
"""Force initial inverted index build. Called after build_index completes on startup."""
if any(vdata.get("files") for vdata in index.values()):
_inverted_index.rebuild()
logger.info("Inverted index initialized.")
def get_inverted_index() -> InvertedIndex:
"""Return the singleton inverted index, rebuilding if stale."""
if _inverted_index.is_stale():
"""Return the singleton inverted index.
Auto-builds on first call if the index has files but the inverted
index hasn't been built yet (fallback for paths that don't go through
``init_inverted_index()``).
"""
if _inverted_index.doc_count == 0 and any(
vdata.get("files") for vdata in index.values()
):
_inverted_index.rebuild()
return _inverted_index