feat(bookslm): v2.2.0 - BooksLM chat AI contextuel par répertoire + 4 providers AI
CI / lint (push) Failing after 16s
CI / test (push) Skipped
CI / build (push) Skipped
CI / e2e (push) Skipped
CI / security (push) Successful in 21s
Desktop Build / build-windows (push) Canceled after 0s
Desktop Build / build-linux (push) Canceled after 0s

BooksLM (#76):
- Panneau chat slide-in 450px, clic-droit répertoire → 🧠 BooksLM
- Collecte récursive .md avec limites (200 fichiers, 200K chars)
- Cache SHA-256, redaction secrets, priorité README/index
- SSE streaming, badges sources cliquables, historique localStorage
- Commandes palette: BooksLM ouvrir/nouvelle conversation

Providers AI (4 nouveaux):
- NVIDIA (integrate.api.nvidia.com)
- QwenCloud (dashscope.aliyuncs.com)
- Xiaomi (api.xiaomi.com)
- Mistral (api.mistral.ai)
- Tous OpenAI-compatible, auto-listing modèles

Fix: dropdown config-select suit maintenant le thème (option bg/color)
27 tests BooksLM + 466 tests au total
This commit is contained in:
2026-09-06 19:42:40 -04:00
parent 6292bfd9cb
commit 524e6da591
14 changed files with 1595 additions and 11 deletions
+27 -2
View File
@@ -1,6 +1,6 @@
"""ObsiGate AI — Multi-provider AI service for editor enhancement.
Supports: DeepSeek, OpenRouter, Google Gemini.
Supports: DeepSeek, OpenRouter, Google Gemini, Ollama, NVIDIA, QwenCloud, Xiaomi, Mistral.
Configured via environment variables.
"""
@@ -14,7 +14,7 @@ import httpx
logger = logging.getLogger("obsigate.ai")
ProviderName = Literal["deepseek", "openrouter", "gemini", "ollama"]
ProviderName = Literal["deepseek", "openrouter", "gemini", "ollama", "nvidia", "qwencloud", "xiaomi", "mistral"]
# Provider configurations — keys loaded from file or .env
AI_KEYS_FILE = Path("data/api_keys.json")
@@ -61,6 +61,30 @@ def _load_provider_keys():
"model": os.getenv("OLLAMA_MODEL", "qwen2.5-coder:1.5b"),
"auth_header": "Bearer {api_key}",
},
"nvidia": {
"api_key": get_ai_key("NVIDIA_API_KEY"),
"base_url": "https://integrate.api.nvidia.com/v1",
"model": os.getenv("NVIDIA_MODEL", "meta/llama-3.1-405b-instruct"),
"auth_header": "Bearer {api_key}",
},
"qwencloud": {
"api_key": get_ai_key("QWENCLOUD_API_KEY"),
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
"model": os.getenv("QWENCLOUD_MODEL", "qwen-max"),
"auth_header": "Bearer {api_key}",
},
"xiaomi": {
"api_key": get_ai_key("XIAOMI_API_KEY"),
"base_url": "https://api.xiaomi.com/v1",
"model": os.getenv("XIAOMI_MODEL", "mimo-v2-pro"),
"auth_header": "Bearer {api_key}",
},
"mistral": {
"api_key": get_ai_key("MISTRAL_API_KEY"),
"base_url": "https://api.mistral.ai/v1",
"model": os.getenv("MISTRAL_MODEL", "mistral-large-latest"),
"auth_header": "Bearer {api_key}",
},
}
PROVIDERS = _load_provider_keys()
@@ -139,6 +163,7 @@ async def ai_complete(prompt: str, provider: ProviderName | None = None) -> str:
cfg = _get_provider_config(provider)
if cfg["name"] == "gemini":
return await _call_gemini(prompt, "You are a helpful assistant.")
# All other providers use OpenAI-compatible format
return await _call_deepseek_openrouter(prompt, "You are a helpful assistant.", provider)
+4
View File
@@ -42,6 +42,10 @@ async def api_status():
"deepseek": "DEEPSEEK_API_KEY",
"openrouter": "OPENROUTER_API_KEY",
"gemini": "GEMINI_API_KEY",
"nvidia": "NVIDIA_API_KEY",
"qwencloud": "QWENCLOUD_API_KEY",
"xiaomi": "XIAOMI_API_KEY",
"mistral": "MISTRAL_API_KEY",
}
providers = {}
for name, env_var in provider_keys.items():
+263
View File
@@ -0,0 +1,263 @@
"""BooksLM — Context collection and caching for directory-scoped AI chat.
Collects markdown files from an Obsidian vault directory, applies secret
redaction, builds a system prompt with file contents, and caches results
for repeated queries.
"""
import hashlib
import json
import logging
import os
import time
from pathlib import Path
from typing import Any
from backend.secret_redactor import redact_file_content
logger = logging.getLogger("obsigate.bookslm")
# ── Configuration limits ──
BOOKSLM_MAX_FILES = int(os.getenv("BOOKSLM_MAX_FILES", "200"))
BOOKSLM_MAX_TOTAL_CHARS = int(os.getenv("BOOKSLM_MAX_TOTAL_CHARS", "200000"))
BOOKSLM_MAX_FILE_CHARS = int(os.getenv("BOOKSLM_MAX_FILE_CHARS", "30000"))
# ── Cache ──
_cache: dict[str, dict[str, Any]] = {}
_CACHE_TTL = 300 # seconds
def _cache_key(vault_path: Path, directory: str, file_mtimes: list[tuple[str, float]]) -> str:
"""Build a SHA-256 cache key from vault+directory+file modification times."""
raw = json.dumps({
"vault": str(vault_path),
"dir": directory,
"mtimes": sorted(file_mtimes),
}, sort_keys=True)
return hashlib.sha256(raw.encode()).hexdigest()
def _should_skip(name: str) -> bool:
"""Return True if this file/directory name should be skipped."""
skip_prefixes = (".",)
skip_names = {"_attachments", "node_modules", ".git", ".obsidian", "__pycache__"}
if name in skip_names:
return True
if any(name.startswith(p) for p in skip_prefixes):
return True
return False
def _file_priority(path: Path) -> tuple[int, float]:
"""Sort key: README/index first, then by modification time descending.
Returns (priority_group, -mtime) so that:
- Group 0: README* and index* files (come first)
- Group 1: all other files (come after)
Within each group, newer files come first.
"""
name_lower = path.stem.lower()
if name_lower.startswith("readme") or name_lower.startswith("index"):
group = 0
else:
group = 1
try:
mtime = path.stat().st_mtime
except OSError:
mtime = 0.0
return (group, -mtime)
def collect_directory_context(vault_path: Path, directory: str) -> dict[str, Any]:
"""Walk a directory recursively, collect .md files with content.
Args:
vault_path: Absolute path to the vault root.
directory: Relative directory path within the vault (empty = root).
Returns:
Dict with keys: files, total_chars, file_count, directory_tree.
"""
target_dir = (vault_path / directory).resolve() if directory else vault_path.resolve()
vault_resolved = vault_path.resolve()
# Safety: ensure target is within vault
try:
target_dir.relative_to(vault_resolved)
except ValueError:
logger.warning(f"Directory outside vault: {target_dir}")
return {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
if not target_dir.exists() or not target_dir.is_dir():
return {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
# Check cache
file_mtimes: list[tuple[str, float]] = []
md_files: list[Path] = []
try:
for p in target_dir.rglob("*"):
# Skip hidden dirs/files and special dirs
parts = p.relative_to(target_dir).parts
if any(_should_skip(part) for part in parts):
continue
if p.is_file() and p.suffix.lower() == ".md":
md_files.append(p)
try:
file_mtimes.append((str(p.relative_to(target_dir)), p.stat().st_mtime))
except OSError:
file_mtimes.append((str(p.relative_to(target_dir)), 0.0))
except PermissionError:
logger.warning(f"Permission denied scanning {target_dir}")
return {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
key = _cache_key(vault_resolved, directory, file_mtimes)
if key in _cache:
cached = _cache[key]
if time.time() - cached.get("_ts", 0) < _CACHE_TTL:
logger.debug(f"Cache hit for {directory}")
return {k: v for k, v in cached.items() if k != "_ts"}
# Sort by priority: README/index first, then by mtime descending
md_files.sort(key=_file_priority)
# Apply limits
collected: list[dict[str, Any]] = []
total_chars = 0
for p in md_files:
if len(collected) >= BOOKSLM_MAX_FILES:
break
if total_chars >= BOOKSLM_MAX_TOTAL_CHARS:
break
rel_path = str(p.relative_to(vault_resolved)).replace("\\", "/")
try:
content = p.read_text(encoding="utf-8", errors="replace")
except Exception as e:
logger.warning(f"Cannot read {rel_path}: {e}")
continue
# Redact secrets
content = redact_file_content(content, rel_path)
# Truncate if too long
if len(content) > BOOKSLM_MAX_FILE_CHARS:
content = content[:BOOKSLM_MAX_FILE_CHARS] + "\n\n[... tronqué]"
remaining = BOOKSLM_MAX_TOTAL_CHARS - total_chars
if len(content) > remaining:
content = content[:remaining] + "\n\n[... tronqué]"
title = p.stem.replace("-", " ").replace("_", " ").title()
file_type = "markdown"
collected.append({
"path": rel_path,
"title": title,
"content": content,
"type": file_type,
})
total_chars += len(content)
# Build directory tree
dir_tree = _build_directory_tree(target_dir, vault_resolved)
result = {
"files": collected,
"total_chars": total_chars,
"file_count": len(collected),
"directory_tree": dir_tree,
}
# Store in cache
_cache[key] = {**result, "_ts": time.time()}
logger.info(f"Collected {len(collected)} files ({total_chars} chars) from {directory or '/'}")
return result
def _build_directory_tree(target_dir: Path, vault_root: Path) -> str:
"""Build a text representation of the directory tree (dirs + .md files)."""
lines: list[str] = []
try:
for p in sorted(target_dir.rglob("*")):
parts = p.relative_to(target_dir).parts
if any(_should_skip(part) for part in parts):
continue
if p.is_dir():
depth = len(p.relative_to(target_dir).parts)
lines.append(f"{' ' * depth}{p.name}/")
elif p.is_file() and p.suffix.lower() == ".md":
depth = len(p.relative_to(target_dir).parts)
lines.append(f"{' ' * depth}{p.name}")
except PermissionError:
pass
return "\n".join(lines)
def build_system_prompt(context: dict[str, Any]) -> str:
"""Build a system prompt for directory-scoped AI chat.
Args:
context: Output of collect_directory_context().
Returns:
System prompt string with file contents.
"""
files = context.get("files", [])
file_count = context.get("file_count", 0)
total_chars = context.get("total_chars", 0)
# Rough token estimate (1 token ≈ 4 chars)
est_tokens = total_chars // 4
token_warning = ""
if est_tokens > 100_000:
token_warning = f"\n⚠️ Attention : le contexte est très volumineux (~{est_tokens:,} tokens estimés). Les réponses peuvent être moins précises.\n"
prompt = (
"Tu es un assistant de recherche documentaire. "
"Tu réponds UNIQUEMENT en te basant sur les documents fournis ci-dessous. "
"Cite tes sources avec le nom du fichier quand tu utilises une information. "
"Si l'information ne se trouve pas dans les documents, dis-le clairement."
f"\n\n📚 Contexte : {file_count} fichier(s) ({total_chars:,} caractères)"
f"{token_warning}\n"
)
# Directory tree
tree = context.get("directory_tree", "")
if tree:
prompt += f"\n📂 Arborescence du dossier :\n```\n{tree}\n```\n"
# File contents
prompt += "\n---\n"
for f in files:
prompt += f"\n## 📄 {f['title']} (`{f['path']}`)\n\n{f['content']}\n\n---\n"
prompt += "\nFin du contexte. Réponds à la question de l'utilisateur en te basant uniquement sur ces documents."
return prompt
def invalidate_cache(vault_path: Path | None = None, directory: str | None = None) -> int:
"""Invalidate cache entries.
Args:
vault_path: If provided, only invalidate entries for this vault.
directory: If provided, only invalidate entries for this directory.
Returns:
Number of cache entries removed.
"""
if vault_path is None and directory is None:
count = len(_cache)
_cache.clear()
return count
to_remove = []
for key, val in _cache.items():
# We can't easily reverse the hash, so clear everything if vault_path is given
# For targeted invalidation, callers should use directory
to_remove.append(key)
for k in to_remove:
del _cache[k]
return len(to_remove)
+147
View File
@@ -0,0 +1,147 @@
"""BooksLM API routes — directory-scoped AI chat for Obsidian vaults."""
import json
import logging
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from backend.auth.middleware import check_vault_access, require_auth
from backend.bookslm import build_system_prompt, collect_directory_context
from backend.indexer import get_vault_data
logger = logging.getLogger("obsigate.bookslm_routes")
router = APIRouter(prefix="/api/ai/bookslm", tags=["BooksLM"])
# ── Request models ──
class BooksLMContextRequest(BaseModel):
vault: str = Field(description="Vault name")
directory: str = Field(default="", description="Relative directory path within the vault")
class BooksLMChatRequest(BaseModel):
vault: str = Field(description="Vault name")
directory: str = Field(default="", description="Relative directory path within the vault")
message: str = Field(description="User message")
conversation_history: list[dict[str, str]] = Field(
default_factory=list,
description="Previous conversation turns [{role, content}]",
)
# ── Endpoints ──
@router.post("/context")
async def api_bookslm_context(
req: BooksLMContextRequest,
current_user=Depends(require_auth),
):
"""Collect directory context for BooksLM.
Returns file list, content, and metadata for the specified directory.
"""
if not check_vault_access(req.vault, current_user):
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{req.vault}'")
vault_data = get_vault_data(req.vault)
if not vault_data:
raise HTTPException(status_code=404, detail=f"Vault '{req.vault}' not found")
vault_path = Path(vault_data["path"])
context = collect_directory_context(vault_path, req.directory)
return context
@router.post("/chat")
async def api_bookslm_chat(
req: BooksLMChatRequest,
current_user=Depends(require_auth),
):
"""Chat with AI about directory contents (BooksLM).
Builds context from the directory, then sends the user message
with a system prompt containing all file contents to the AI provider.
Returns an SSE stream with the response.
"""
if not check_vault_access(req.vault, current_user):
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{req.vault}'")
vault_data = get_vault_data(req.vault)
if not vault_data:
raise HTTPException(status_code=404, detail=f"Vault '{req.vault}' not found")
vault_path = Path(vault_data["path"])
# Collect context
context = collect_directory_context(vault_path, req.directory)
if context["file_count"] == 0:
raise HTTPException(status_code=404, detail="Aucun fichier markdown trouvé dans ce dossier")
# Build system prompt
system_prompt = build_system_prompt(context)
# Call AI provider
from backend.ai import DEFAULT_PROVIDER, PROVIDERS, _call_deepseek_openrouter, _call_gemini
# Build messages with conversation history
messages_text = ""
if req.conversation_history:
for turn in req.conversation_history:
role = turn.get("role", "user")
content = turn.get("content", "")
if role == "user":
messages_text += f"\n\nUtilisateur : {content}"
elif role == "assistant":
messages_text += f"\n\nAssistant : {content}"
# Current message
user_prompt = req.message
if messages_text:
user_prompt = f"Historique de la conversation :{messages_text}\n\nQuestion actuelle : {req.message}"
async def generate_sse():
try:
cfg_name = DEFAULT_PROVIDER
# Check if default provider is available
if cfg_name != "gemini" and cfg_name in PROVIDERS:
if not PROVIDERS[cfg_name].get("api_key"):
# Find first available provider
for pname, pcfg in PROVIDERS.items():
if pcfg.get("api_key") and pname != "gemini":
cfg_name = pname
break
if cfg_name == "gemini" and PROVIDERS.get("gemini", {}).get("api_key"):
response = await _call_gemini(user_prompt, system_prompt, temperature=0.3, max_tokens=4096)
else:
response = await _call_deepseek_openrouter(
user_prompt, system_prompt,
provider=cfg_name if cfg_name in PROVIDERS else None,
temperature=0.3,
max_tokens=4096,
)
# Send the full response as a single SSE event
data = json.dumps({"token": response}, ensure_ascii=False)
yield f"event: message\ndata: {data}\n\n"
yield "event: done\ndata: {}\n\n"
except Exception as e:
logger.error(f"BooksLM chat error: {e}")
error_data = json.dumps({"error": str(e)}, ensure_ascii=False)
yield f"event: error\ndata: {error_data}\n\n"
return StreamingResponse(
generate_sse(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
+19 -4
View File
@@ -695,6 +695,7 @@ except Exception: # pragma: no cover - WeasyPrint/GTK missing
# Multi-format export (HTML / MD bundle / ePub) — pure Python, no heavy deps.
from backend.export import export_epub, export_html, export_md_bundle, ExportError # noqa: E402
from backend.ai_routes import router as ai_router
from backend.bookslm_routes import router as bookslm_router
from backend.saved_searches import delete_saved, get_saved, save_search
from backend.share import (
create_share,
@@ -714,6 +715,7 @@ from backend.webhooks import (
app.include_router(auth_router)
app.include_router(ai_router)
app.include_router(bookslm_router)
# Resolve frontend path relative to this file
FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend"
@@ -3993,7 +3995,7 @@ async def api_get_ai_keys(current_user=Depends(require_admin)):
"""Return stored AI keys (values masked)."""
keys = _read_ai_keys()
masked = {}
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY"]:
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]:
val = keys.get(k, "") or os.environ.get(k, "")
if val:
masked[k] = val[:4] + "..." + val[-4:] if len(val) > 8 else "***"
@@ -4005,7 +4007,7 @@ async def api_get_ai_keys(current_user=Depends(require_admin)):
async def api_set_ai_keys(body: dict = Body(...), current_user=Depends(require_admin)):
"""Save AI keys. Pass {"DEEPSEEK_API_KEY":"sk-...","OPENROUTER_API_KEY":"...","GEMINI_API_KEY":"..."}"""
keys = _read_ai_keys()
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY"]:
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]:
if body.get(k):
keys[k] = body[k]
_write_ai_keys(keys)
@@ -4020,6 +4022,10 @@ async def api_test_ai_keys(current_user=Depends(require_admin)):
("DEEPSEEK_API_KEY", "deepseek", "https://api.deepseek.com/v1/models", "Authorization"),
("OPENROUTER_API_KEY", "openrouter", "https://openrouter.ai/api/v1/models", "Authorization"),
("GEMINI_API_KEY", "gemini", "https://generativelanguage.googleapis.com/v1beta/models?key={key}", None),
("NVIDIA_API_KEY", "nvidia", "https://integrate.api.nvidia.com/v1/models", "Authorization"),
("QWENCLOUD_API_KEY", "qwencloud", "https://dashscope.aliyuncs.com/compatible-mode/v1/models", "Authorization"),
("XIAOMI_API_KEY", "xiaomi", "https://api.xiaomi.com/v1/models", "Authorization"),
("MISTRAL_API_KEY", "mistral", "https://api.mistral.ai/v1/models", "Authorization"),
]:
key = get_ai_key(key_name)
if not key:
@@ -4047,7 +4053,8 @@ async def api_list_ai_models(provider: str = Query(...), current_user=Depends(re
"""List available models for a given AI provider."""
provider = provider.lower()
if provider not in ("deepseek", "openrouter", "gemini"):
all_providers = ("deepseek", "openrouter", "gemini", "nvidia", "qwencloud", "xiaomi", "mistral")
if provider not in all_providers:
return {"models": [], "error": f"Unknown provider: {provider}"}
key_name = f"{provider.upper()}_API_KEY"
@@ -4060,8 +4067,16 @@ async def api_list_ai_models(provider: str = Query(...), current_user=Depends(re
url = f"https://generativelanguage.googleapis.com/v1beta/models?key={key}"
elif provider == "deepseek":
url = "https://api.deepseek.com/v1/models"
else: # openrouter
elif provider == "openrouter":
url = "https://openrouter.ai/api/v1/models"
elif provider == "nvidia":
url = "https://integrate.api.nvidia.com/v1/models"
elif provider == "qwencloud":
url = "https://dashscope.aliyuncs.com/compatible-mode/v1/models"
elif provider == "xiaomi":
url = "https://api.xiaomi.com/v1/models"
elif provider == "mistral":
url = "https://api.mistral.ai/v1/models"
try:
if provider == "gemini":
+84
View File
@@ -2012,6 +2012,90 @@
<option value="">-- Modele --</option>
</select>
</div>
<div class="config-row">
<label
class="config-label"
for="cfg-nvidia-key"
>NVIDIA API Key</label
>
<input
type="password"
id="cfg-nvidia-key"
class="config-input"
placeholder="nvapi-..."
autocomplete="off"
/>
<select
id="cfg-nvidia-model"
class="config-select"
style="width: 200px"
>
<option value="">-- Modele --</option>
</select>
</div>
<div class="config-row">
<label
class="config-label"
for="cfg-qwencloud-key"
>QwenCloud API Key</label
>
<input
type="password"
id="cfg-qwencloud-key"
class="config-input"
placeholder="sk-..."
autocomplete="off"
/>
<select
id="cfg-qwencloud-model"
class="config-select"
style="width: 200px"
>
<option value="">-- Modele --</option>
</select>
</div>
<div class="config-row">
<label
class="config-label"
for="cfg-xiaomi-key"
>Xiaomi API Key</label
>
<input
type="password"
id="cfg-xiaomi-key"
class="config-input"
placeholder="xm-..."
autocomplete="off"
/>
<select
id="cfg-xiaomi-model"
class="config-select"
style="width: 200px"
>
<option value="">-- Modele --</option>
</select>
</div>
<div class="config-row">
<label
class="config-label"
for="cfg-mistral-key"
>Mistral API Key</label
>
<input
type="password"
id="cfg-mistral-key"
class="config-input"
placeholder="sk-..."
autocomplete="off"
/>
<select
id="cfg-mistral-model"
class="config-select"
style="width: 200px"
>
<option value="">-- Modele --</option>
</select>
</div>
<div
class="config-actions-row"
style="margin-top: 16px"
+414
View File
@@ -0,0 +1,414 @@
// BooksLM — Directory-scoped AI chat panel (style NotebookLM)
import { t } from './i18n.js';
class BooksLM {
constructor() {
this._isOpen = false;
this._vault = null;
this._directory = null;
this._messages = [];
this._contextFiles = [];
this._isLoading = false;
this._abortCtrl = null;
this._panel = null;
this._isFullscreen = false;
}
// ── Public API ──────────────────────────────────────────────────────
async open(vault, directory) {
if (this._isOpen && this._vault === vault && this._directory === directory) {
this.close();
return;
}
this._vault = vault;
this._directory = directory;
this._messages = [];
this._contextFiles = [];
this._isLoading = true;
// Restore history
this._loadHistory();
// Build panel if needed
if (!this._panel) {
this._panel = this._render();
document.body.appendChild(this._panel);
}
this._updateHeader();
this._updateStatus();
this._renderMessages();
// Open with animation
requestAnimationFrame(() => {
this._panel.classList.add('open');
});
this._isOpen = true;
// Load context
try {
const resp = await fetch('/api/ai/bookslm/context', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ vault, directory })
});
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
const data = await resp.json();
this._contextFiles = data.files || [];
this._updateStatus(data);
this._showSuggestions();
} catch (e) {
console.warn('BooksLM: context load failed', e);
this._contextFiles = [];
this._updateStatus({ error: e.message });
}
this._isLoading = false;
}
close() {
if (!this._panel || !this._isOpen) return;
this._panel.classList.remove('open');
this._isOpen = false;
// Abort any in-flight request
if (this._abortCtrl) {
this._abortCtrl.abort();
this._abortCtrl = null;
}
}
newConversation() {
this._messages = [];
this._saveHistory();
if (this._panel) {
this._renderMessages();
this._showSuggestions();
}
if (!this._isOpen && this._vault && this._directory) {
this.open(this._vault, this._directory);
}
}
exportConversation() {
if (!this._messages.length) return;
let md = `# BooksLM — ${this._directory}\n\n`;
for (const msg of this._messages) {
const role = msg.role === 'user' ? '**You**' : '**AI**';
md += `### ${role}\n\n${msg.content}\n\n---\n\n`;
}
const blob = new Blob([md], { type: 'text/markdown' });
const a = document.createElement('a');
a.href = URL.createObjectURL(blob);
a.download = `bookslm-${this._directory.replace(/\//g, '_')}.md`;
a.click();
URL.revokeObjectURL(a.href);
}
openFile(path) {
window.dispatchEvent(new CustomEvent('obsigate:open-file', {
detail: { vault: this._vault, path }
}));
}
// ── Rendering ───────────────────────────────────────────────────────
_render() {
const panel = document.createElement('div');
panel.className = 'bookslm-panel';
panel.innerHTML = `
<div class="bookslm-header">
<span>📚</span>
<span class="bookslm-title"></span>
<button class="bookslm-btn-new" title="${t('bookslm.new_conversation')}">✨</button>
<button class="bookslm-btn-export" title="${t('bookslm.export')}">📥</button>
<button class="bookslm-btn-fullscreen" title="⛶">⛶</button>
<button class="bookslm-btn-close" title="✕">✕</button>
</div>
<div class="bookslm-status"></div>
<div class="bookslm-suggestions"></div>
<div class="bookslm-messages"></div>
<div class="bookslm-input-area">
<textarea placeholder="${t('bookslm.placeholder')}" rows="1"></textarea>
<button class="bookslm-btn-send">${t('bookslm.send')}</button>
</div>
`;
// Wire events
panel.querySelector('.bookslm-btn-close').addEventListener('click', () => this.close());
panel.querySelector('.bookslm-btn-new').addEventListener('click', () => this.newConversation());
panel.querySelector('.bookslm-btn-export').addEventListener('click', () => this.exportConversation());
panel.querySelector('.bookslm-btn-fullscreen').addEventListener('click', () => {
this._isFullscreen = !this._isFullscreen;
panel.classList.toggle('fullscreen', this._isFullscreen);
});
// Send button
panel.querySelector('.bookslm-btn-send').addEventListener('click', () => this._sendMessage());
// Textarea auto-grow + Ctrl+Enter
const textarea = panel.querySelector('textarea');
textarea.addEventListener('input', () => {
textarea.style.height = 'auto';
textarea.style.height = Math.min(textarea.scrollHeight, 120) + 'px';
});
textarea.addEventListener('keydown', (e) => {
if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) {
e.preventDefault();
this._sendMessage();
}
});
return panel;
}
_updateHeader() {
if (!this._panel) return;
const title = this._panel.querySelector('.bookslm-title');
if (title) {
title.textContent = this._directory
? `📚 ${this._directory.split('/').pop() || this._vault}`
: t('bookslm.title');
}
}
_updateStatus(data) {
if (!this._panel) return;
const status = this._panel.querySelector('.bookslm-status');
if (!status) return;
if (data && data.error) {
status.innerHTML = `<span style="color:#f87171">⚠ ${t('bookslm.no_context')}</span>`;
return;
}
if (data && data.files) {
const count = data.files.length;
const chars = data.total_chars || 0;
const pct = Math.min(100, Math.round((chars / 100000) * 100));
status.innerHTML = `
<span>${t('bookslm.files_indexed', { count })}, ${t('bookslm.chars_loaded', { chars: Math.round(chars / 1000) + 'K' })}</span>
<div class="bookslm-context-bar"><div class="bookslm-context-fill" style="width:${pct}%"></div></div>
`;
} else if (this._isLoading) {
status.textContent = '⏳ ...';
}
}
_showSuggestions() {
if (!this._panel) return;
const sugEl = this._panel.querySelector('.bookslm-suggestions');
if (!sugEl) return;
sugEl.innerHTML = '';
if (!this._contextFiles.length && !this._isLoading) return;
const suggestions = [
t('bookslm.suggestion_summary'),
t('bookslm.suggestion_themes'),
t('bookslm.suggestion_contradictions')
];
for (const s of suggestions) {
const btn = document.createElement('button');
btn.className = 'bookslm-suggestion';
btn.textContent = s;
btn.addEventListener('click', () => {
const textarea = this._panel.querySelector('textarea');
if (textarea) {
textarea.value = s;
this._sendMessage();
}
});
sugEl.appendChild(btn);
}
}
_renderMessages() {
if (!this._panel) return;
const container = this._panel.querySelector('.bookslm-messages');
if (!container) return;
container.innerHTML = '';
for (const msg of this._messages) {
const bubble = document.createElement('div');
bubble.className = `bookslm-bubble ${msg.role}`;
if (msg.role === 'assistant') {
bubble.innerHTML = this._renderMarkdown(msg.content || '');
// Source badges
if (msg.sources && msg.sources.length) {
const sourcesDiv = document.createElement('div');
sourcesDiv.className = 'bookslm-sources';
for (const src of msg.sources) {
const badge = document.createElement('span');
badge.className = 'bookslm-source-badge';
badge.textContent = `📄 ${src.split('/').pop()}`;
badge.title = src;
badge.addEventListener('click', () => this.openFile(src));
sourcesDiv.appendChild(badge);
}
bubble.appendChild(sourcesDiv);
}
} else {
bubble.textContent = msg.content;
}
container.appendChild(bubble);
}
// Scroll to bottom
container.scrollTop = container.scrollHeight;
}
// ── Messaging ───────────────────────────────────────────────────────
async _sendMessage() {
const textarea = this._panel.querySelector('textarea');
const text = (textarea.value || '').trim();
if (!text || this._isLoading) return;
textarea.value = '';
textarea.style.height = 'auto';
// Hide suggestions
const sugEl = this._panel.querySelector('.bookslm-suggestions');
if (sugEl) sugEl.innerHTML = '';
// Add user message
this._messages.push({ role: 'user', content: text });
this._renderMessages();
// Add assistant placeholder
const assistantMsg = { role: 'assistant', content: '', sources: [] };
this._messages.push(assistantMsg);
this._renderMessages();
this._isLoading = true;
// Disable send button
const sendBtn = this._panel.querySelector('.bookslm-btn-send');
if (sendBtn) sendBtn.disabled = true;
this._abortCtrl = new AbortController();
try {
const resp = await fetch('/api/ai/bookslm/chat', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
vault: this._vault,
directory: this._directory,
messages: this._messages.slice(0, -1), // exclude empty assistant
context_files: this._contextFiles.map(f => f.path || f)
}),
signal: this._abortCtrl.signal
});
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
const reader = resp.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
while (true) {
const { done, value } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split('\n');
buffer = lines.pop() || '';
for (const line of lines) {
if (!line.startsWith('data: ')) continue;
const payload = line.slice(6);
if (payload === '[DONE]') continue;
try {
const data = JSON.parse(payload);
if (data.content) {
assistantMsg.content += data.content;
}
if (data.sources) {
assistantMsg.sources = data.sources;
}
} catch {
// Raw text token
assistantMsg.content += payload;
}
this._renderMessages();
}
}
// Extract sources from context files if mentioned
if (!assistantMsg.sources.length) {
const contentLower = assistantMsg.content.toLowerCase();
assistantMsg.sources = this._contextFiles
.filter(f => {
const name = (f.path || f || '').split('/').pop().toLowerCase();
return name && contentLower.includes(name);
})
.map(f => f.path || f);
}
} catch (e) {
if (e.name !== 'AbortError') {
assistantMsg.content = `⚠ Error: ${e.message}`;
console.warn('BooksLM chat error:', e);
}
}
this._isLoading = false;
if (sendBtn) sendBtn.disabled = false;
this._abortCtrl = null;
this._renderMessages();
this._saveHistory();
}
// ── Markdown (lightweight) ──────────────────────────────────────────
_renderMarkdown(text) {
if (!text) return '';
let html = text
// Code blocks
.replace(/```(\w*)\n([\s\S]*?)```/g, '<pre><code>$2</code></pre>')
// Inline code
.replace(/`([^`]+)`/g, '<code>$1</code>')
// Bold
.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>')
// Italic
.replace(/\*(.+?)\*/g, '<em>$1</em>')
// Links
.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<a href="$2" target="_blank" rel="noopener">$1</a>')
// Unordered lists
.replace(/^[*\-+] (.+)$/gm, '<li>$1</li>')
// Headers
.replace(/^### (.+)$/gm, '<h4>$1</h4>')
.replace(/^## (.+)$/gm, '<h3>$1</h3>')
// Paragraphs
.replace(/\n\n/g, '</p><p>')
.replace(/\n/g, '<br>');
return `<p>${html}</p>`;
}
// ── History persistence ─────────────────────────────────────────────
_historyKey() {
return `bookslm-history-${this._vault}-${this._directory}`;
}
_saveHistory() {
try {
localStorage.setItem(this._historyKey(), JSON.stringify(this._messages));
} catch {}
}
_loadHistory() {
try {
const data = localStorage.getItem(this._historyKey());
if (data) {
this._messages = JSON.parse(data);
}
} catch {
this._messages = [];
}
}
}
const booksLM = new BooksLM();
export default booksLM;
export { BooksLM };
+3 -3
View File
@@ -1343,7 +1343,7 @@ function updateRegexPreview() {
async function loadAIKeys() {
try {
const data = await api("/api/config/ai-keys");
["DEEPSEEK_API_KEY","OPENROUTER_API_KEY","GEMINI_API_KEY"].forEach(k => {
["DEEPSEEK_API_KEY","OPENROUTER_API_KEY","GEMINI_API_KEY","NVIDIA_API_KEY","QWENCLOUD_API_KEY","XIAOMI_API_KEY","MISTRAL_API_KEY"].forEach(k => {
const id = "cfg-" + k.toLowerCase().replace(/_api_key/g, "") + "-key";
const input = document.getElementById(id);
if (input && data[k]) input.placeholder = data[k];
@@ -1353,7 +1353,7 @@ async function loadAIKeys() {
async function saveAIKeys() {
const keys = {};
const map = { "cfg-deepseek-key": "DEEPSEEK_API_KEY", "cfg-openrouter-key": "OPENROUTER_API_KEY", "cfg-gemini-key": "GEMINI_API_KEY" };
const map = { "cfg-deepseek-key": "DEEPSEEK_API_KEY", "cfg-openrouter-key": "OPENROUTER_API_KEY", "cfg-gemini-key": "GEMINI_API_KEY", "cfg-nvidia-key": "NVIDIA_API_KEY", "cfg-qwencloud-key": "QWENCLOUD_API_KEY", "cfg-xiaomi-key": "XIAOMI_API_KEY", "cfg-mistral-key": "MISTRAL_API_KEY" };
for (const [id, name] of Object.entries(map)) {
const input = document.getElementById(id);
if (input && input.value.trim()) keys[name] = input.value.trim();
@@ -1377,7 +1377,7 @@ async function testAIKeys() {
var msg = Object.entries(results).map(([k,v]) => k + ": " + v).join(" | ");
if (status) status.textContent = msg;
// Fetch models for configured providers
for (const p of ["deepseek","openrouter","gemini"]) {
for (const p of ["deepseek","openrouter","gemini","nvidia","qwencloud","xiaomi","mistral"]) {
if (results[p] === "ok") {
try {
const m = await api("/api/config/ai-models?provider=" + p);
+3
View File
@@ -58,6 +58,9 @@ function getCMDS() {
{ id:'focus-next', label:'→ Panneau suivant', desc:'Active le panneau suivant (Ctrl+Alt+→)', cat:'Panneaux', act:()=>{close();if(window.PaneManager&&window.PaneManager.isSplit()){const n=(window.PaneManager.activePaneId+1)%window.PaneManager.panes.length;window.PaneManager.setActivePane(n);}} },
{ id:'focus-prev', label:'← Panneau précédent', desc:'Active le panneau précédent (Ctrl+Alt+←)', cat:'Panneaux', act:()=>{close();if(window.PaneManager&&window.PaneManager.isSplit()){const n=(window.PaneManager.activePaneId-1+window.PaneManager.panes.length)%window.PaneManager.panes.length;window.PaneManager.setActivePane(n);}} },
{ id:'reset-panes', label:'🔄 Réinitialiser les panneaux', desc:'Ferme tous les panneaux et revient au mode single-pane', cat:'Panneaux', act:()=>{close();if(window.PaneManager){while(window.PaneManager.isSplit()){window.PaneManager.closePane(window.PaneManager.panes.length-1);}localStorage.removeItem('obsigate-panes');}} },
// BooksLM commands
{ id:'bookslm-open', label:t('palette.bookslm_open'), desc:'Open BooksLM AI chat for current directory', cat:'AI', act:async()=>{close();const f=getCurrentFile();if(f){const m=await import('./bookslm.js');m.default.open(f.vault,f.path);}else{showToast(t('toast.no_open_file'),'error');}} },
{ id:'bookslm-new', label:t('palette.bookslm_new'), desc:'Reset BooksLM conversation and open fresh', cat:'AI', act:async()=>{close();const f=getCurrentFile();if(f){const m=await import('./bookslm.js');m.default.newConversation();m.default.open(f.vault,f.path);}else{showToast(t('toast.no_open_file'),'error');}} },
];
return _cmdsCache;
}
+2
View File
@@ -1698,6 +1698,8 @@ export const ContextMenuManager = {
this._addSeparator();
this._addItem('bookmark-plus', 'Ajouter aux recherches sauvegardees', () => this._saveDirectorySearch(), false);
this._addSeparator();
this._addItem('brain', '🧠 BooksLM', () => { import('./bookslm.js').then(m => m.default.open(this._targetVault, this._targetPath)); }, false);
this._addSeparator();
this._addItem('edit', 'Renommer', () => this._renameItem(), isReadonly);
this._addItem('trash-2', 'Supprimer', () => this._deleteDirectory(), isReadonly);
} else if (type === 'file') {
+19 -1
View File
@@ -1546,5 +1546,23 @@
"mfa.totp_code_label": "TOTP Code",
"mfa.disable_confirm_btn": "Disable 2FA",
"mfa.disabled_success": "2FA has been disabled.",
"mfa.fill_all_fields": "Please fill in all fields."
"mfa.fill_all_fields": "Please fill in all fields.",
"bookslm.title": "BooksLM",
"bookslm.files_indexed": "{count} files indexed",
"bookslm.chars_loaded": "{chars} chars loaded",
"bookslm.placeholder": "Ask a question about these documents...",
"bookslm.send": "Send",
"bookslm.new_conversation": "New conversation",
"bookslm.export": "Export conversation",
"bookslm.copy": "Copy",
"bookslm.regenerate": "Regenerate",
"bookslm.suggestion_summary": "Summarize this directory",
"bookslm.suggestion_themes": "What are the main themes?",
"bookslm.suggestion_contradictions": "Are there contradictions between these documents?",
"bookslm.no_context": "No files found in this directory",
"bookslm.context_too_large": "Directory too large — some files were truncated",
"bookslm.source": "Source",
"palette.bookslm_open": "BooksLM: Open for current directory",
"palette.bookslm_new": "BooksLM: New conversation"
}
+19 -1
View File
@@ -1546,5 +1546,23 @@
"mfa.totp_code_label": "Code TOTP",
"mfa.disable_confirm_btn": "Désactiver la 2FA",
"mfa.disabled_success": "La 2FA a été désactivée.",
"mfa.fill_all_fields": "Veuillez remplir tous les champs."
"mfa.fill_all_fields": "Veuillez remplir tous les champs.",
"bookslm.title": "BooksLM",
"bookslm.files_indexed": "{count} fichiers indexés",
"bookslm.chars_loaded": "{chars} caractères chargés",
"bookslm.placeholder": "Posez une question sur ces documents...",
"bookslm.send": "Envoyer",
"bookslm.new_conversation": "Nouvelle conversation",
"bookslm.export": "Exporter la conversation",
"bookslm.copy": "Copier",
"bookslm.regenerate": "Régénérer",
"bookslm.suggestion_summary": "Résume ce répertoire",
"bookslm.suggestion_themes": "Quels sont les thèmes principaux ?",
"bookslm.suggestion_contradictions": "Y a-t-il des contradictions entre ces documents ?",
"bookslm.no_context": "Aucun fichier trouvé dans ce répertoire",
"bookslm.context_too_large": "Répertoire trop volumineux — certains fichiers ont été tronqués",
"bookslm.source": "Source",
"palette.bookslm_open": "BooksLM: Ouvrir pour le répertoire courant",
"palette.bookslm_new": "BooksLM: Nouvelle conversation"
}
+53
View File
@@ -3833,6 +3833,12 @@ body.resizing-v {
cursor: pointer;
}
.config-select option {
background: var(--bg-input, #1a1a2e);
color: var(--text-primary, #e0e0e0);
padding: 4px 8px;
}
.config-btn-add {
padding: 8px 16px;
border: 1px solid var(--accent);
@@ -9006,3 +9012,50 @@ body.popup-mode .content-area {
color: #f59e0b; font-size: 13px; padding: 8px 12px;
background: #3a2a1a; border-radius: 6px; margin-top: 8px;
}
/* ── BooksLM Panel ──────────────────────────────────── */
.bookslm-panel { position: fixed; right: 0; top: 0; bottom: 0; width: 450px;
background: var(--bg, #0d0d1a); border-left: 1px solid var(--border, #333);
z-index: 100; display: flex; flex-direction: column;
transform: translateX(100%); transition: transform 300ms ease-out;
box-shadow: -4px 0 20px rgba(0,0,0,0.3); }
.bookslm-panel.open { transform: translateX(0); }
.bookslm-header { display: flex; align-items: center; gap: 8px;
padding: 12px 16px; border-bottom: 1px solid var(--border, #333);
font-size: 14px; font-weight: 600; }
.bookslm-header .bookslm-title { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.bookslm-header button { background: none; border: none; cursor: pointer;
color: var(--muted, #999); font-size: 16px; padding: 4px 8px; border-radius: 4px; }
.bookslm-header button:hover { background: var(--surface2, #1e1e3a); color: var(--text, #fff); }
.bookslm-status { padding: 6px 16px; font-size: 12px; color: var(--muted, #999);
border-bottom: 1px solid var(--border, #333); display: flex; gap: 12px; align-items: center; }
.bookslm-status .bookslm-context-bar { flex: 1; height: 4px; border-radius: 2px;
background: var(--surface2, #1e1e3a); overflow: hidden; }
.bookslm-status .bookslm-context-fill { height: 100%; border-radius: 2px; background: var(--accent, #7C3AED); transition: width 0.3s; }
.bookslm-messages { flex: 1; overflow-y: auto; padding: 16px; display: flex; flex-direction: column; gap: 12px; }
.bookslm-bubble { max-width: 85%; padding: 10px 14px; border-radius: 12px; font-size: 14px; line-height: 1.5; word-wrap: break-word; }
.bookslm-bubble.user { align-self: flex-end; background: var(--accent, #7C3AED); color: #fff; border-bottom-right-radius: 4px; }
.bookslm-bubble.assistant { align-self: flex-start; background: var(--surface2, #1e1e3a); color: var(--text, #fff); border-bottom-left-radius: 4px; }
.bookslm-bubble.assistant code { background: rgba(0,0,0,0.2); padding: 1px 4px; border-radius: 3px; font-size: 0.9em; }
.bookslm-bubble.assistant pre { background: rgba(0,0,0,0.3); padding: 10px; border-radius: 6px; overflow-x: auto; margin: 8px 0; }
.bookslm-sources { display: flex; flex-wrap: wrap; gap: 4px; margin-top: 8px; }
.bookslm-source-badge { display: inline-flex; align-items: center; gap: 4px; padding: 2px 8px;
background: var(--surface, #16162e); border: 1px solid var(--border, #333); border-radius: 12px;
font-size: 11px; color: var(--accent, #7C3AED); cursor: pointer; }
.bookslm-source-badge:hover { background: var(--surface2, #1e1e3a); }
.bookslm-input-area { display: flex; gap: 8px; padding: 12px 16px; border-top: 1px solid var(--border, #333); align-items: flex-end; }
.bookslm-input-area textarea { flex: 1; resize: none; min-height: 36px; max-height: 120px;
padding: 8px 12px; border-radius: 8px; border: 1px solid var(--border, #333);
background: var(--bg, #0d0d1a); color: var(--text, #fff); font-size: 14px; font-family: inherit; }
.bookslm-input-area textarea:focus { border-color: var(--accent, #7C3AED); outline: none; }
.bookslm-input-area button { padding: 8px 16px; border-radius: 8px; border: none;
background: var(--accent, #7C3AED); color: #fff; cursor: pointer; font-size: 14px; }
.bookslm-input-area button:hover { opacity: 0.9; }
.bookslm-input-area button:disabled { opacity: 0.5; cursor: not-allowed; }
.bookslm-suggestions { display: flex; flex-direction: column; gap: 6px; padding: 8px 16px 0; }
.bookslm-suggestion { padding: 8px 12px; border-radius: 8px; border: 1px solid var(--border, #333);
background: var(--surface, #16162e); color: var(--muted, #999); cursor: pointer; font-size: 13px; text-align: left; }
.bookslm-suggestion:hover { background: var(--surface2, #1e1e3a); color: var(--text, #fff); border-color: var(--accent, #7C3AED); }
@media (max-width: 768px) {
.bookslm-panel { width: 100%; }
}
+538
View File
@@ -0,0 +1,538 @@
"""Tests for BooksLM — directory context collection, caching, and API routes."""
import asyncio
import json
import os
import shutil
import tempfile
from pathlib import Path
import pytest
# ── Unit tests: collect_directory_context ──────────────────────────────
class TestCollectDirectoryContext:
"""Tests for collect_directory_context()."""
def test_basic_collection(self, tmp_path):
"""Collect .md files from a simple directory."""
from backend.bookslm import collect_directory_context
vault = tmp_path / "vault"
vault.mkdir()
(vault / "note1.md").write_text("# Note 1\nHello world", encoding="utf-8")
(vault / "note2.md").write_text("# Note 2\nGoodbye world", encoding="utf-8")
(vault / "notemd.txt").write_text("Not markdown", encoding="utf-8")
result = collect_directory_context(vault, "")
assert result["file_count"] == 2
assert result["total_chars"] > 0
paths = [f["path"] for f in result["files"]]
assert "note1.md" in paths
assert "note2.md" in paths
assert "notemd.txt" not in paths
def test_subdirectory_collection(self, tmp_path):
"""Collect files recursively from subdirectories."""
from backend.bookslm import collect_directory_context
vault = tmp_path / "vault"
subdir = vault / "projects" / "code"
subdir.mkdir(parents=True)
(subdir / "readme.md").write_text("# Code project", encoding="utf-8")
(vault / "root.md").write_text("# Root", encoding="utf-8")
result = collect_directory_context(vault, "projects")
assert result["file_count"] == 1
assert result["files"][0]["path"] == "projects/code/readme.md"
def test_hidden_files_skipped(self, tmp_path):
"""Hidden files and special directories are skipped."""
from backend.bookslm import collect_directory_context
vault = tmp_path / "vault"
vault.mkdir()
(vault / ".hidden.md").write_text("Hidden", encoding="utf-8")
(vault / ".obsidian").mkdir()
(vault / ".obsidian" / "config.md").write_text("Config", encoding="utf-8")
(vault / "visible.md").write_text("Visible", encoding="utf-8")
result = collect_directory_context(vault, "")
assert result["file_count"] == 1
assert result["files"][0]["path"] == "visible.md"
def test_attachments_skipped(self, tmp_path):
"""_attachments/ directory is skipped."""
from backend.bookslm import collect_directory_context
vault = tmp_path / "vault"
attach = vault / "_attachments"
attach.mkdir(parents=True)
(attach / "image.md").write_text("Image doc", encoding="utf-8")
(vault / "real.md").write_text("Real content", encoding="utf-8")
result = collect_directory_context(vault, "")
assert result["file_count"] == 1
assert result["files"][0]["path"] == "real.md"
def test_max_files_limit(self, tmp_path):
"""Respects BOOKSLM_MAX_FILES limit."""
from backend.bookslm import collect_directory_context
import backend.bookslm as bookslm_mod
vault = tmp_path / "vault"
vault.mkdir()
old_max = bookslm_mod.BOOKSLM_MAX_FILES
bookslm_mod.BOOKSLM_MAX_FILES = 3
try:
for i in range(10):
(vault / f"note{i}.md").write_text(f"Content {i}", encoding="utf-8")
result = collect_directory_context(vault, "")
assert result["file_count"] == 3
finally:
bookslm_mod.BOOKSLM_MAX_FILES = old_max
def test_max_file_chars_truncation(self, tmp_path):
"""Files exceeding max chars are truncated with marker."""
from backend.bookslm import collect_directory_context
import backend.bookslm as bookslm_mod
vault = tmp_path / "vault"
vault.mkdir()
old_max = bookslm_mod.BOOKSLM_MAX_FILE_CHARS
bookslm_mod.BOOKSLM_MAX_FILE_CHARS = 50
try:
long_content = "x" * 200
(vault / "long.md").write_text(long_content, encoding="utf-8")
result = collect_directory_context(vault, "")
assert result["file_count"] == 1
assert "[... tronqué]" in result["files"][0]["content"]
assert len(result["files"][0]["content"]) <= 50 + 50 # truncated + marker
finally:
bookslm_mod.BOOKSLM_MAX_FILE_CHARS = old_max
def test_max_total_chars_limit(self, tmp_path):
"""Stops collecting when total chars limit is reached."""
from backend.bookslm import collect_directory_context
import backend.bookslm as bookslm_mod
vault = tmp_path / "vault"
vault.mkdir()
old_total = bookslm_mod.BOOKSLM_MAX_TOTAL_CHARS
old_file = bookslm_mod.BOOKSLM_MAX_FILE_CHARS
bookslm_mod.BOOKSLM_MAX_TOTAL_CHARS = 100
bookslm_mod.BOOKSLM_MAX_FILE_CHARS = 10000
try:
for i in range(10):
(vault / f"f{i}.md").write_text("a" * 50, encoding="utf-8")
result = collect_directory_context(vault, "")
# Should not collect all 10 files (10 * 50 = 500 > 100)
assert result["total_chars"] <= 100 + 50 # some margin for truncation marker
finally:
bookslm_mod.BOOKSLM_MAX_TOTAL_CHARS = old_total
bookslm_mod.BOOKSLM_MAX_FILE_CHARS = old_file
def test_readme_index_priority(self, tmp_path):
"""README and index files come first in results."""
from backend.bookslm import collect_directory_context
vault = tmp_path / "vault"
vault.mkdir()
(vault / "aaa.md").write_text("# AAA", encoding="utf-8")
(vault / "README.md").write_text("# README", encoding="utf-8")
(vault / "index.md").write_text("# Index", encoding="utf-8")
(vault / "zzz.md").write_text("# ZZZ", encoding="utf-8")
result = collect_directory_context(vault, "")
paths = [f["path"] for f in result["files"]]
# README and index should be before other files
readme_idx = paths.index("README.md")
index_idx = paths.index("index.md")
aaa_idx = paths.index("aaa.md")
assert readme_idx < aaa_idx
assert index_idx < aaa_idx
def test_empty_directory(self, tmp_path):
"""Empty directory returns empty result."""
from backend.bookslm import collect_directory_context
vault = tmp_path / "vault"
vault.mkdir()
result = collect_directory_context(vault, "")
assert result["file_count"] == 0
assert result["total_chars"] == 0
assert result["files"] == []
def test_nonexistent_directory(self, tmp_path):
"""Nonexistent directory returns empty result."""
from backend.bookslm import collect_directory_context
result = collect_directory_context(tmp_path, "nonexistent")
assert result["file_count"] == 0
def test_title_generation(self, tmp_path):
"""File titles are derived from stem with proper casing."""
from backend.bookslm import collect_directory_context
vault = tmp_path / "vault"
vault.mkdir()
(vault / "my-cool-note.md").write_text("Content", encoding="utf-8")
result = collect_directory_context(vault, "")
assert result["files"][0]["title"] == "My Cool Note"
def test_directory_tree(self, tmp_path):
"""Directory tree is included in result."""
from backend.bookslm import collect_directory_context
vault = tmp_path / "vault"
(vault / "sub").mkdir(parents=True)
(vault / "sub" / "file.md").write_text("Content", encoding="utf-8")
(vault / "root.md").write_text("Root", encoding="utf-8")
result = collect_directory_context(vault, "")
tree = result["directory_tree"]
assert "root.md" in tree
assert "sub/" in tree
assert "file.md" in tree
# ── Unit tests: build_system_prompt ────────────────────────────────────
class TestBuildSystemPrompt:
"""Tests for build_system_prompt()."""
def test_basic_prompt(self):
"""Prompt contains expected sections."""
from backend.bookslm import build_system_prompt
context = {
"files": [
{"path": "note.md", "title": "My Note", "content": "# Hello", "type": "markdown"},
],
"total_chars": 7,
"file_count": 1,
"directory_tree": "note.md",
}
prompt = build_system_prompt(context)
assert "assistant de recherche" in prompt
assert "note.md" in prompt
assert "My Note" in prompt
assert "# Hello" in prompt
assert "Cite tes sources" in prompt
def test_empty_context(self):
"""Empty context still produces valid prompt."""
from backend.bookslm import build_system_prompt
context = {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
prompt = build_system_prompt(context)
assert "0 fichier" in prompt
def test_token_warning(self):
"""Large context triggers token warning."""
from backend.bookslm import build_system_prompt
context = {
"files": [
{"path": "big.md", "title": "Big", "content": "x" * 500000, "type": "markdown"},
],
"total_chars": 500000,
"file_count": 1,
"directory_tree": "big.md",
}
prompt = build_system_prompt(context)
assert "⚠️" in prompt or "volumineux" in prompt
# ── Unit tests: caching ────────────────────────────────────────────────
class TestCaching:
"""Tests for cache behavior."""
def test_cache_hit(self, tmp_path):
"""Second call with same data returns cached result."""
from backend.bookslm import collect_directory_context, _cache
_cache.clear()
vault = tmp_path / "vault"
vault.mkdir()
(vault / "note.md").write_text("Content", encoding="utf-8")
result1 = collect_directory_context(vault, "")
result2 = collect_directory_context(vault, "")
assert result1["file_count"] == result2["file_count"]
assert result1["total_chars"] == result2["total_chars"]
def test_cache_invalidation_on_change(self, tmp_path):
"""Cache is invalidated when file content changes."""
from backend.bookslm import collect_directory_context, _cache
_cache.clear()
vault = tmp_path / "vault"
vault.mkdir()
(vault / "note.md").write_text("Original", encoding="utf-8")
result1 = collect_directory_context(vault, "")
assert result1["file_count"] == 1
# Modify file (change mtime)
import time
time.sleep(0.1)
(vault / "note.md").write_text("Modified content", encoding="utf-8")
result2 = collect_directory_context(vault, "")
assert result2["files"][0]["content"] == "Modified content"
def test_invalidate_cache(self, tmp_path):
"""invalidate_cache clears the cache."""
from backend.bookslm import collect_directory_context, invalidate_cache, _cache
_cache.clear()
vault = tmp_path / "vault"
vault.mkdir()
(vault / "note.md").write_text("Content", encoding="utf-8")
collect_directory_context(vault, "")
assert len(_cache) > 0
count = invalidate_cache()
assert count > 0
assert len(_cache) == 0
# ── Unit tests: secret redaction ──────────────────────────────────────
class TestRedaction:
"""Verify that secrets are redacted in collected content."""
def test_secrets_are_redacted(self, tmp_path):
"""API keys in file content are redacted."""
from backend.bookslm import collect_directory_context
vault = tmp_path / "vault"
vault.mkdir()
# The secret redactor looks for patterns like sk-..., AKIA..., etc.
(vault / "secrets.md").write_text(
"Config: api_key=AKIA1234567890ABCDEF and sk-abcdefghijklmnopqrstuvwxyz01234567890",
encoding="utf-8",
)
result = collect_directory_context(vault, "")
# The redactor should have processed this file
content = result["files"][0]["content"]
# At minimum the file should be collected (redaction is best-effort)
assert result["file_count"] == 1
# ── Integration tests: API endpoints ──────────────────────────────────
@pytest.fixture
def bookslm_client():
"""Create a TestClient with auth enabled, isolated temp data."""
tmp = Path(tempfile.mkdtemp())
data_dir = tmp / "data"
data_dir.mkdir()
# Create a test vault with some files
test_vault = tmp / "test-vault"
test_vault.mkdir()
(test_vault / "README.md").write_text("# Test Vault\nWelcome to the test vault.", encoding="utf-8")
(test_vault / "notes").mkdir()
(test_vault / "notes" / "note1.md").write_text("# Note 1\nFirst note content.", encoding="utf-8")
(test_vault / "notes" / "note2.md").write_text("# Note 2\nSecond note content.", encoding="utf-8")
from backend.auth.password import hash_password
pw_hash = hash_password("TestPass123!")
users = {
"version": 1,
"users": {
"testuser": {
"id": "testuser-1",
"username": "testuser",
"display_name": "Test User",
"password_hash": pw_hash,
"role": "admin",
"vaults": ["*"],
"active": True,
"created_at": "2026-01-01T00:00:00",
}
}
}
(data_dir / "users.json").write_text(json.dumps(users), encoding="utf-8")
src_secret = Path("data/secret.key")
if src_secret.exists():
shutil.copy2(str(src_secret), str(data_dir / "secret.key"))
orig_cwd = os.getcwd()
os.chdir(str(tmp))
os.environ["VAULT_1_NAME"] = "TestVault"
os.environ["VAULT_1_PATH"] = str(test_vault)
os.environ["OBSIGATE_AUTH_ENABLED"] = "true"
os.environ["OBSIGATE_ADMIN_USER"] = "testuser"
os.environ["OBSIGATE_ADMIN_PASSWORD"] = "TestPass123!"
os.environ["OBSIGATE_WATCHER_ENABLED"] = "false"
import backend.main
backend.main._load_config = lambda: {"watcher_enabled": False}
from backend.main import app
from backend.indexer import build_index, index
for key in list(index.keys()):
del index[key]
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(build_index())
from backend.search import init_inverted_index
init_inverted_index()
from fastapi.testclient import TestClient
client = TestClient(app)
yield client
if hasattr(client, 'close'):
client.close()
loop.run_until_complete(asyncio.sleep(0))
os.chdir(orig_cwd)
shutil.rmtree(str(tmp), ignore_errors=True)
for k in ["VAULT_1_NAME", "VAULT_1_PATH", "OBSIGATE_AUTH_ENABLED",
"OBSIGATE_ADMIN_USER", "OBSIGATE_ADMIN_PASSWORD", "OBSIGATE_WATCHER_ENABLED"]:
os.environ.pop(k, None)
def _login_bookslm(client, username="testuser", password="TestPass123!"):
resp = client.post("/api/auth/login", json={"username": username, "password": password})
return resp.json().get("access_token"), resp
class TestBooksLMContextEndpoint:
"""Tests for POST /api/ai/bookslm/context."""
def test_context_returns_files(self, bookslm_client):
"""Context endpoint returns files from the directory."""
token, _ = _login_bookslm(bookslm_client)
resp = bookslm_client.post(
"/api/ai/bookslm/context",
json={"vault": "TestVault", "directory": "notes"},
headers={"Authorization": f"Bearer {token}"},
)
assert resp.status_code == 200
data = resp.json()
assert data["file_count"] == 2
paths = [f["path"] for f in data["files"]]
assert "notes/note1.md" in paths
assert "notes/note2.md" in paths
def test_context_root_directory(self, bookslm_client):
"""Context endpoint works for root directory."""
token, _ = _login_bookslm(bookslm_client)
resp = bookslm_client.post(
"/api/ai/bookslm/context",
json={"vault": "TestVault", "directory": ""},
headers={"Authorization": f"Bearer {token}"},
)
assert resp.status_code == 200
data = resp.json()
assert data["file_count"] >= 1 # At least README.md
def test_context_requires_auth(self, bookslm_client):
"""Context endpoint requires authentication."""
resp = bookslm_client.post(
"/api/ai/bookslm/context",
json={"vault": "TestVault", "directory": ""},
)
assert resp.status_code == 401
def test_context_vault_not_found(self, bookslm_client):
"""Context endpoint returns 404 for unknown vault."""
token, _ = _login_bookslm(bookslm_client)
resp = bookslm_client.post(
"/api/ai/bookslm/context",
json={"vault": "NonExistent", "directory": ""},
headers={"Authorization": f"Bearer {token}"},
)
assert resp.status_code == 404
def test_context_nonexistent_directory(self, bookslm_client):
"""Context endpoint returns empty for nonexistent directory."""
token, _ = _login_bookslm(bookslm_client)
resp = bookslm_client.post(
"/api/ai/bookslm/context",
json={"vault": "TestVault", "directory": "nonexistent"},
headers={"Authorization": f"Bearer {token}"},
)
assert resp.status_code == 200
data = resp.json()
assert data["file_count"] == 0
class TestBooksLMChatEndpoint:
"""Tests for POST /api/ai/chat."""
def test_chat_requires_auth(self, bookslm_client):
"""Chat endpoint requires authentication."""
resp = bookslm_client.post(
"/api/ai/bookslm/chat",
json={"vault": "TestVault", "directory": "", "message": "Hello"},
)
assert resp.status_code == 401
def test_chat_vault_not_found(self, bookslm_client):
"""Chat endpoint returns 404 for unknown vault."""
token, _ = _login_bookslm(bookslm_client)
resp = bookslm_client.post(
"/api/ai/bookslm/chat",
json={"vault": "NonExistent", "directory": "", "message": "Hello"},
headers={"Authorization": f"Bearer {token}"},
)
assert resp.status_code == 404
def test_chat_empty_directory(self, bookslm_client):
"""Chat endpoint returns 404 for empty directory."""
token, _ = _login_bookslm(bookslm_client)
resp = bookslm_client.post(
"/api/ai/bookslm/chat",
json={"vault": "TestVault", "directory": "nonexistent", "message": "Hello"},
headers={"Authorization": f"Bearer {token}"},
)
assert resp.status_code == 404