Projets/ObsiGate
1
1
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects Releases 19 Wiki Activity

feat(search): recherche semantique - embeddings vectoriels + hybride RRF (#70)

Browse Source
This commit is contained in:
bruno
2026-09-12 00:12:20 -04:00
parent 45be3125de
commit 31d4be8015
19 changed files with 1388 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/search.py
+171 -54
View File
@@ -4,12 +4,14 @@ import re
import time
import unicodedata
from collections import defaultdict
from collections.abc import Callable
from typing import Any
from snowballstemmer import stemmer as _snowball_stemmer
from sortedcontainers import SortedList
from backend import indexer as _indexer
from backend import semantic_search as _semantic
from backend.indexer import index
logger = logging.getLogger("obsigate.search")
@@ -655,6 +657,10 @@ def _on_index_change_hook(action: str, vault_name: str, path: str, file_info: di
inv.remove_document(vault_name, path)
except Exception as e:
logger.warning(f"Inverted index incremental update failed ({action} {vault_name}/{path}): {e}")
try:
_semantic.on_index_change(action, vault_name, path, file_info)
except Exception as e:
logger.warning(f"Semantic index incremental update failed ({action} {vault_name}/{path}): {e}")
# Register the hook with indexer (indexer is already imported at top of file)
@@ -1102,6 +1108,7 @@ def advanced_search(
created: str | None = None,
modified: str | None = None,
size: str | None = None,
semantic: bool = False,
) -> dict[str, Any]:
"""Advanced full-text search with TF-IDF scoring, facets, and pagination.
@@ -1121,10 +1128,13 @@ def advanced_search(
limit: Max results per page.
offset: Pagination offset.
sort_by: ``"relevance"`` or ``"modified"``.
semantic: When True, fuse the TF-IDF ranking with the semantic
(embedding) ranking via Reciprocal Rank Fusion and expose a
``semantic_score`` per result.
Returns:
Dict with ``results``, ``total``, ``offset``, ``limit``, ``facets``,
``query_time_ms``.
``query_time_ms`` and ``semantic_available``.
"""
t0 = time.monotonic()
query = query.strip() if query else ""
@@ -1182,58 +1192,63 @@ def advanced_search(
# ------------------------------------------------------------------
# Step 2: Apply filters on candidate set
# ------------------------------------------------------------------
if effective_vault != "all":
candidates &= inv.vault_docs.get(effective_vault, set())
if all_tags and has_terms:
for t in all_tags:
candidates &= inv.tag_docs.get(t.lower(), set())
if parsed["title"]:
norm_title_filter = normalize_text(parsed["title"])
candidates = {
dk for dk in candidates
if norm_title_filter in normalize_text(inv.doc_info[dk].get("title", ""))
}
if parsed["path"]:
norm_path_filter = normalize_text(parsed["path"])
candidates = {
dk for dk in candidates
if norm_path_filter in normalize_text(inv.doc_info[dk].get("path", ""))
}
if parsed["ext"]:
ext_filter = parsed["ext"]
candidates = {
dk for dk in candidates
if (
inv.doc_info[dk].get("path", "").rsplit("/", 1)[-1].lower() == ext_filter
or inv.doc_info[dk].get("path", "").rsplit("/", 1)[-1].lower().endswith(f".{ext_filter}")
)
}
# Date and size filters (from query operators or API params)
date_range_created = _parse_date_range(created or parsed.get("created"))
if date_range_created:
candidates = {
dk for dk in candidates
if _matches_date_range(inv.doc_info[dk].get("created"), date_range_created)
}
date_range_modified = _parse_date_range(modified or parsed.get("modified"))
if date_range_modified:
candidates = {
dk for dk in candidates
if _matches_date_range(inv.doc_info[dk].get("modified"), date_range_modified)
}
size_range = _parse_size_range(size or parsed.get("size"))
if size_range:
candidates = {
dk for dk in candidates
if _matches_size_range(inv.doc_info[dk].get("size", 0), size_range)
}
def _apply_metadata_filters(docs: set) -> set:
"""Restrict a document-key set to the query's metadata filters."""
if effective_vault != "all":
docs &= inv.vault_docs.get(effective_vault, set())
if all_tags:
for t in all_tags:
docs &= inv.tag_docs.get(t.lower(), set())
if parsed["title"]:
norm_title_filter = normalize_text(parsed["title"])
docs = {
dk for dk in docs
if norm_title_filter in normalize_text(inv.doc_info[dk].get("title", ""))
}
if parsed["path"]:
norm_path_filter = normalize_text(parsed["path"])
docs = {
dk for dk in docs
if norm_path_filter in normalize_text(inv.doc_info[dk].get("path", ""))
}
if parsed["ext"]:
ext_filter = parsed["ext"]
docs = {
dk for dk in docs
if (
inv.doc_info[dk].get("path", "").rsplit("/", 1)[-1].lower() == ext_filter
or inv.doc_info[dk].get("path", "").rsplit("/", 1)[-1].lower().endswith(f".{ext_filter}")
)
}
if date_range_created:
docs = {
dk for dk in docs
if _matches_date_range(inv.doc_info[dk].get("created"), date_range_created)
}
if date_range_modified:
docs = {
dk for dk in docs
if _matches_date_range(inv.doc_info[dk].get("modified"), date_range_modified)
}
if size_range:
docs = {
dk for dk in docs
if _matches_size_range(inv.doc_info[dk].get("size", 0), size_range)
}
return docs
candidates = _apply_metadata_filters(candidates)
# ------------------------------------------------------------------
# Step 3: Score only the candidates (not all N documents)
@@ -1312,16 +1327,22 @@ def advanced_search(
"title": file_info["title"],
"tags": file_info.get("tags", []),
"score": round(score, 4),
"semantic_score": 0.0,
"snippet": snippet,
"modified": file_info.get("modified", ""),
"extension": file_info.get("extension", file_info.get("path", "").rsplit(".", 1)[-1] if "." in file_info.get("path", "") else ""),
}
scored_results.append((score, result))
# Facets
facet_vaults[vault_name] = facet_vaults.get(vault_name, 0) + 1
for tag in file_info.get("tags", []):
facet_tags[tag] = facet_tags.get(tag, 0) + 1
# ------------------------------------------------------------------
# Step 4: Optional semantic fusion (RRF with the TF-IDF ranking)
# ------------------------------------------------------------------
semantic_available = _semantic.get_semantic_index().is_ready()
if semantic and has_terms and not regex:
scored_results = _fuse_semantic_results(
scored_results, query, effective_vault, inv, _apply_metadata_filters,
include_paths, exclude_paths, limit,
)
# Sort
if sort_by == "modified":
@@ -1329,6 +1350,12 @@ def advanced_search(
else:
scored_results.sort(key=lambda x: -x[0])
# Facets are recomputed from the final result set (covers semantic-only docs)
for _, result in scored_results:
facet_vaults[result["vault"]] = facet_vaults.get(result["vault"], 0) + 1
for tag in result.get("tags", []):
facet_tags[tag] = facet_tags.get(tag, 0) + 1
total = len(scored_results)
page = scored_results[offset: offset + limit]
elapsed_ms = round((time.monotonic() - t0) * 1000, 1)
@@ -1343,9 +1370,99 @@ def advanced_search(
"vaults": dict(sorted(facet_vaults.items(), key=lambda x: -x[1])),
},
"query_time_ms": elapsed_ms,
"semantic_available": semantic_available,
}
def _fuse_semantic_results(
scored_results: list[tuple[float, dict[str, Any]]],
query: str,
vault_filter: str,
inv: InvertedIndex,
apply_metadata_filters: Callable[[set], set],
include_paths: str | None,
exclude_paths: str | None,
limit: int,
) -> list[tuple[float, dict[str, Any]]]:
"""Fuse the TF-IDF ranking with the semantic ranking using RRF.
Documents found only by the semantic engine are materialized from the
inverted index metadata (with a plain, non-highlighted snippet). The
returned tuples carry the fused score, and every result dict gets a
``semantic_score`` (cosine similarity, 0.0 when absent).
Args:
scored_results: Existing ``(tfidf_score, result_dict)`` tuples.
query: Raw free-text query.
vault_filter: Effective vault filter.
inv: Inverted index (document metadata source).
apply_metadata_filters: Callable restricting a doc-key set to the
query's tag/title/path/ext/date/size filters.
include_paths: Include glob patterns (or None).
exclude_paths: Exclude glob patterns (or None).
limit: Requested page size (drives how many semantic hits to fetch).
Returns:
New ``(fused_score, result_dict)`` list (unsorted).
"""
sem_hits = _semantic.semantic_search_docs(query, vault_filter=vault_filter, top_k=max(limit * 5, 200))
if not sem_hits:
return scored_results
# Semantic candidates must satisfy the same metadata + path filters.
semantic_universe = apply_metadata_filters(set(inv.doc_info.keys()))
semantic_universe = {
dk for dk in semantic_universe
if _passes_path_filters(inv.doc_info[dk].get("path", ""), include_paths, exclude_paths)
}
sem_scores: dict[str, float] = {}
sem_ranked: list[str] = []
for doc_key, similarity in sem_hits:
if doc_key not in semantic_universe:
continue
sem_scores[doc_key] = similarity
sem_ranked.append(doc_key)
if not sem_ranked:
return scored_results
existing: dict[str, dict[str, Any]] = {}
lexical_ranked: list[str] = []
for _, lex_result in sorted(scored_results, key=lambda item: -item[0]):
key = f"{lex_result['vault']}::{lex_result['path']}"
existing[key] = lex_result
lexical_ranked.append(key)
fused = _semantic.rrf_fuse([lexical_ranked, sem_ranked])
merged: list[tuple[float, dict[str, Any]]] = []
for doc_key, fused_score in fused.items():
result = existing.get(doc_key)
if result is None:
file_info = inv.doc_info.get(doc_key)
if file_info is None:
continue
content = file_info.get("content", "")
result = {
"vault": inv.doc_vault[doc_key],
"path": file_info["path"],
"title": file_info["title"],
"tags": file_info.get("tags", []),
"score": 0.0,
"semantic_score": 0.0,
"snippet": _escape_html(content[:200].strip()) if content else "",
"modified": file_info.get("modified", ""),
"extension": file_info.get(
"extension",
file_info.get("path", "").rsplit(".", 1)[-1] if "." in file_info.get("path", "") else "",
),
}
result["semantic_score"] = round(sem_scores.get(doc_key, 0.0), 4)
merged.append((fused_score, result))
return merged
# ---------------------------------------------------------------------------
# Suggestion helpers
# ---------------------------------------------------------------------------