feat(search): recherche semantique - embeddings vectoriels + hybride RRF (#70)

tests/conftest.py

@@ -114,6 +114,10 @@ def app_with_vault(test_vault_dir: str):
     from backend.search import init_inverted_index
     init_inverted_index()
 
+    # Build semantic (embedding) index — uses the dependency-free hash fallback
+    from backend.semantic_search import init_semantic_index
+    init_semantic_index()
+
     client = TestClient(app)
     return client
 
@@ -196,6 +200,9 @@ def admin_client(tmp_path):
     from backend.search import init_inverted_index
     init_inverted_index()
 
+    from backend.semantic_search import init_semantic_index
+    init_semantic_index()
+
     from fastapi.testclient import TestClient
     client = TestClient(app)
     yield client

tests/frontend/semantic-search.test.mjs

@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+/**
+ * ObsiGate - JSDOM tests for the semantic search toggle (ROADMAP #70).
+ *
+ * Verifies the "Recherche sémantique" toggle wiring in frontend/js/search.js:
+ *   - the toggle is disabled until the backend reports semantic availability
+ *   - clicking it flips state.semanticSearch and the active class
+ *   - clicking while unavailable is a no-op
+ *
+ * Usage: node tests/frontend/semantic-search.test.mjs
+ */
+
+import { strict as assert } from "node:assert";
+import { JSDOM } from "jsdom";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import path from "node:path";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const REPO_ROOT = path.resolve(__dirname, "..", "..");
+
+// ── JSDOM bootstrap ────────────────────────────────────────────────────────
+const dom = new JSDOM(
+  `<!DOCTYPE html><html><body>
+    <input id="search-input" type="text" />
+    <select id="vault-filter"><option value="all">all</option></select>
+    <div id="results-bar" style="display:none">
+      <span id="rb-count">0/0</span>
+      <button id="rb-prev"></button>
+      <button id="rb-next"></button>
+      <button id="rb-case"></button>
+      <button id="rb-word"></button>
+      <button id="rb-regex"></button>
+      <button id="rb-semantic" disabled></button>
+    </div>
+    <button id="search-filter-btn"></button>
+    <button id="search-clear-btn"></button>
+    <div id="search-filter-row" style="display:none">
+      <input id="search-include-input" />
+      <input id="search-exclude-input" />
+    </div>
+    <div id="content-area"></div>
+    <div id="search-chips"></div>
+  </body></html>`,
+  { url: "https://example.com/", pretendToBeVisual: true },
+);
+
+const w = dom.window;
+globalThis.window = w;
+globalThis.document = w.document;
+globalThis.HTMLElement = w.HTMLElement;
+globalThis.Element = w.Element;
+globalThis.Node = w.Node;
+globalThis.Event = w.Event;
+globalThis.CustomEvent = w.CustomEvent;
+globalThis.KeyboardEvent = w.KeyboardEvent;
+globalThis.localStorage = w.localStorage;
+Object.defineProperty(globalThis, "navigator", {
+  value: w.navigator,
+  configurable: true,
+  writable: true,
+});
+globalThis.MutationObserver = w.MutationObserver;
+globalThis.getComputedStyle = w.getComputedStyle.bind(w);
+globalThis.requestAnimationFrame = (cb) => setTimeout(cb, 0);
+globalThis.cancelAnimationFrame = (id) => clearTimeout(id);
+globalThis.fetch = async () => ({ ok: true, json: async () => ({}) });
+
+const stateMod = await import(
+  pathToFileURL(path.join(REPO_ROOT, "frontend", "js", "state.js")).href
+);
+const { state } = stateMod;
+const searchMod = await import(
+  pathToFileURL(path.join(REPO_ROOT, "frontend", "js", "search.js")).href
+);
+
+// ── Test harness ───────────────────────────────────────────────────────────
+let testCount = 0;
+let passCount = 0;
+
+async function test(name, fn) {
+  testCount++;
+  try {
+    await fn();
+    console.log(`  \u2713 ${name}`);
+    passCount++;
+  } catch (e) {
+    console.log(`  \u2717 ${name}`);
+    console.log(`      ${e.message}`);
+  }
+}
+
+searchMod.initSearch();
+const semanticBtn = document.getElementById("rb-semantic");
+
+await test("semantic toggle is disabled until availability is known", () => {
+  assert.equal(semanticBtn.disabled, true);
+});
+
+await test("clicking while unavailable does not toggle", () => {
+  semanticBtn.click();
+  assert.equal(state.semanticSearch, false);
+});
+
+await test("clicking while available toggles state and active class", () => {
+  state.semanticAvailable = true;
+  semanticBtn.disabled = false;
+  semanticBtn.click();
+  assert.equal(state.semanticSearch, true);
+  assert.ok(semanticBtn.classList.contains("active"));
+  semanticBtn.click();
+  assert.equal(state.semanticSearch, false);
+  assert.ok(!semanticBtn.classList.contains("active"));
+});
+
+await test("semantic state keys exist", () => {
+  assert.equal(state.semanticAvailable, true);
+  assert.equal(typeof state.semanticSearch, "boolean");
+});
+
+console.log(`\n${passCount}/${testCount} tests passed`);
+if (passCount !== testCount) process.exit(1);

tests/test_semantic_search.py

@@ -0,0 +1,243 @@
+# tests/test_semantic_search.py — Tests for semantic search (embeddings + RRF)
+import math
+
+from backend.search import advanced_search
+from backend.semantic_search import (
+    EMBEDDING_DIM,
+    HashEmbeddingProvider,
+    SemanticIndex,
+    VectorStore,
+    chunk_text,
+    get_semantic_index,
+    on_index_change,
+    reset_semantic_index,
+    rrf_fuse,
+    semantic_status,
+)
+
+# ═══════════════════════════════════════════════════════════════════
+# Chunking
+# ═══════════════════════════════════════════════════════════════════
+
+class TestChunkText:
+    def test_empty(self):
+        assert chunk_text("") == []
+        assert chunk_text("   ") == []
+
+    def test_short_text_single_chunk(self):
+        chunks = chunk_text("un deux trois", chunk_tokens=512)
+        assert chunks == ["un deux trois"]
+
+    def test_long_text_multiple_chunks(self):
+        words = " ".join(f"mot{i}" for i in range(1200))
+        chunks = chunk_text(words, chunk_tokens=512, overlap=64)
+        assert len(chunks) >= 3
+        # First chunk has 512 words
+        assert len(chunks[0].split()) == 512
+
+    def test_overlap_shared_tokens(self):
+        words = [f"w{i}" for i in range(600)]
+        chunks = chunk_text(" ".join(words), chunk_tokens=100, overlap=20)
+        first_tail = chunks[0].split()[-20:]
+        second_head = chunks[1].split()[:20]
+        assert first_tail == second_head
+
+
+# ═══════════════════════════════════════════════════════════════════
+# Hash embedding provider (dependency-free fallback)
+# ═══════════════════════════════════════════════════════════════════
+
+class TestHashEmbeddingProvider:
+    def setup_method(self):
+        self.provider = HashEmbeddingProvider()
+
+    def test_dimension(self):
+        vec = self.provider.encode_one("hello world")
+        assert len(vec) == EMBEDDING_DIM
+
+    def test_l2_normalized(self):
+        vec = self.provider.encode_one("un texte de test assez long")
+        norm = math.sqrt(sum(v * v for v in vec))
+        assert abs(norm - 1.0) < 1e-6
+
+    def test_empty_text_zero_vector(self):
+        vec = self.provider.encode_one("")
+        assert all(v == 0.0 for v in vec)
+
+    def test_deterministic(self):
+        a = self.provider.encode_one("sauvegarde des données")
+        b = self.provider.encode_one("sauvegarde des données")
+        assert a == b
+
+    def test_similar_more_similar_than_dissimilar(self):
+        base = self.provider.encode_one("stratégie de sauvegarde automatique des données")
+        close = self.provider.encode_one("sauvegarde automatique des données")
+        far = self.provider.encode_one("recette de cuisine au chocolat")
+        sim_close = sum(x * y for x, y in zip(base, close))
+        sim_far = sum(x * y for x, y in zip(base, far))
+        assert sim_close > sim_far
+
+    def test_batch_matches_single(self):
+        texts = ["premier document", "deuxième document"]
+        batch = self.provider.encode(texts)
+        assert batch[0] == self.provider.encode_one(texts[0])
+        assert batch[1] == self.provider.encode_one(texts[1])
+
+
+# ═══════════════════════════════════════════════════════════════════
+# Vector store
+# ═══════════════════════════════════════════════════════════════════
+
+class TestVectorStore:
+    def test_add_and_search(self):
+        provider = HashEmbeddingProvider()
+        store = VectorStore(provider.dimension)
+        store.add("v::a.md", "a", provider.encode_one("python programmation"))
+        store.add("v::b.md", "b", provider.encode_one("recette cuisine chocolat"))
+        hits = store.search(provider.encode_one("python"), top_k=2)
+        assert hits[0][0] == "v::a.md"
+        assert hits[0][1] > hits[1][1]
+
+    def test_remove_document(self):
+        provider = HashEmbeddingProvider()
+        store = VectorStore(provider.dimension)
+        store.add("v::a.md", "a", provider.encode_one("python"))
+        store.add("v::a.md", "a2", provider.encode_one("code"))
+        store.add("v::b.md", "b", provider.encode_one("cuisine"))
+        assert len(store) == 3
+        store.remove_document("v::a.md")
+        assert len(store) == 1
+        hits = store.search(provider.encode_one("python"), top_k=5)
+        assert all(key != "v::a.md" for key, _ in hits)
+
+    def test_empty_search(self):
+        store = VectorStore(EMBEDDING_DIM)
+        assert store.search([0.0] * EMBEDDING_DIM) == []
+
+
+# ═══════════════════════════════════════════════════════════════════
+# Reciprocal Rank Fusion
+# ═══════════════════════════════════════════════════════════════════
+
+class TestRRF:
+    def test_single_ranking(self):
+        scores = rrf_fuse([["a", "b"]])
+        assert scores["a"] > scores["b"]
+
+    def test_fusion_promotes_consensus(self):
+        scores = rrf_fuse([["a", "b", "c"], ["b", "c", "d"]])
+        # "b" is well ranked by both methods -> best fused score
+        assert max(scores, key=scores.get) == "b"
+
+    def test_dedup_within_ranking(self):
+        scores = rrf_fuse([["a", "a", "b"]])
+        single = rrf_fuse([["a", "b"]])
+        assert abs(scores["a"] - single["a"]) < 1e-9
+
+
+# ═══════════════════════════════════════════════════════════════════
+# SemanticIndex — unit (injected provider, manual documents)
+# ═══════════════════════════════════════════════════════════════════
+
+class TestSemanticIndexUnit:
+    def test_add_search_remove(self):
+        index = SemanticIndex(provider=HashEmbeddingProvider())
+        index._ready = True
+        index.add_document("V", "backup.md", {
+            "path": "backup.md",
+            "title": "Stratégie de backup",
+            "content": "Protection et sauvegarde automatique des données",
+            "tags": [],
+        })
+        index.add_document("V", "cuisine.md", {
+            "path": "cuisine.md",
+            "title": "Recette au chocolat",
+            "content": "Faire fondre le chocolat puis ajouter la farine",
+            "tags": [],
+        })
+        hits = index.search("sauvegarde des données", top_k=5)
+        assert hits
+        assert hits[0][0] == "V::backup.md"
+
+        index.remove_document("V", "backup.md")
+        hits = index.search("sauvegarde des données", top_k=5)
+        assert all(key != "V::backup.md" for key, _ in hits)
+
+    def test_vault_filter(self):
+        index = SemanticIndex(provider=HashEmbeddingProvider())
+        index._ready = True
+        index.add_document("V1", "a.md", {"path": "a.md", "title": "python", "content": "python", "tags": []})
+        index.add_document("V2", "b.md", {"path": "b.md", "title": "python", "content": "python", "tags": []})
+        hits = index.search("python", vault_filter="V1", top_k=5)
+        assert hits
+        assert all(key.startswith("V1::") for key, _ in hits)
+
+    def test_not_ready_is_noop(self):
+        index = SemanticIndex(provider=HashEmbeddingProvider())
+        index.add_document("V", "a.md", {"path": "a.md", "title": "x", "content": "x", "tags": []})
+        assert len(index.store) == 0
+        assert index.search("x") == []
+
+
+# ═══════════════════════════════════════════════════════════════════
+# Integration with the global index / advanced_search
+# ═══════════════════════════════════════════════════════════════════
+
+class TestSemanticIntegration:
+    def test_rebuild_from_global_index(self, client):
+        index = get_semantic_index()
+        assert index.is_ready()
+        assert len(index.doc_keys) >= 3
+
+    def test_on_index_change_hook(self, client):
+        index = get_semantic_index()
+        file_info = {
+            "path": "semantic_hook.md",
+            "title": "Sauvegarde",
+            "content": "Sauvegarde automatique des données",
+            "tags": [],
+        }
+        on_index_change("add", "TestVault", "semantic_hook.md", file_info)
+        assert "TestVault::semantic_hook.md" in index.doc_keys
+        on_index_change("remove", "TestVault", "semantic_hook.md", file_info)
+        assert "TestVault::semantic_hook.md" not in index.doc_keys
+
+    def test_advanced_search_semantic(self, client):
+        result = advanced_search("python", vault_filter="all", semantic=True)
+        assert result["semantic_available"] is True
+        assert len(result["results"]) >= 1
+        assert any(r.get("semantic_score", 0) > 0 for r in result["results"])
+
+    def test_advanced_search_semantic_field_default(self, client):
+        result = advanced_search("python", vault_filter="all")
+        for r in result["results"]:
+            assert "semantic_score" in r
+
+    def test_semantic_status(self, client):
+        status = semantic_status()
+        assert status["available"] is True
+        assert status["documents"] >= 3
+        assert status["dimension"] > 0
+
+    def test_reset_semantic_index(self, client):
+        reset_semantic_index()
+        assert get_semantic_index().is_ready() is False
+        # Restore for subsequent tests
+        from backend.semantic_search import init_semantic_index
+        init_semantic_index()
+
+
+class TestSemanticAPI:
+    def test_api_semantic_flag(self, client):
+        resp = client.get("/api/search/advanced?q=python&vault=all&semantic=true")
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["semantic_available"] is True
+        assert len(data["results"]) >= 1
+        assert "semantic_score" in data["results"][0]
+
+    def test_api_without_semantic(self, client):
+        resp = client.get("/api/search/advanced?q=python&vault=all")
+        assert resp.status_code == 200
+        data = resp.json()
+        assert "semantic_available" in data