feat(bookslm): v2.2.0 - BooksLM chat AI contextuel par répertoire + 4 providers AI
CI / lint (push) Failing after 16s
CI / test (push) Skipped
CI / build (push) Skipped
CI / e2e (push) Skipped
CI / security (push) Successful in 21s
Desktop Build / build-windows (push) Canceled after 0s
Desktop Build / build-linux (push) Canceled after 0s

BooksLM (#76):
- Panneau chat slide-in 450px, clic-droit répertoire → 🧠 BooksLM
- Collecte récursive .md avec limites (200 fichiers, 200K chars)
- Cache SHA-256, redaction secrets, priorité README/index
- SSE streaming, badges sources cliquables, historique localStorage
- Commandes palette: BooksLM ouvrir/nouvelle conversation

Providers AI (4 nouveaux):
- NVIDIA (integrate.api.nvidia.com)
- QwenCloud (dashscope.aliyuncs.com)
- Xiaomi (api.xiaomi.com)
- Mistral (api.mistral.ai)
- Tous OpenAI-compatible, auto-listing modèles

Fix: dropdown config-select suit maintenant le thème (option bg/color)
27 tests BooksLM + 466 tests au total
This commit is contained in:
2026-09-06 19:42:40 -04:00
parent 6292bfd9cb
commit 524e6da591
14 changed files with 1595 additions and 11 deletions
+27 -2
View File
@@ -1,6 +1,6 @@
"""ObsiGate AI — Multi-provider AI service for editor enhancement.
Supports: DeepSeek, OpenRouter, Google Gemini.
Supports: DeepSeek, OpenRouter, Google Gemini, Ollama, NVIDIA, QwenCloud, Xiaomi, Mistral.
Configured via environment variables.
"""
@@ -14,7 +14,7 @@ import httpx
logger = logging.getLogger("obsigate.ai")
ProviderName = Literal["deepseek", "openrouter", "gemini", "ollama"]
ProviderName = Literal["deepseek", "openrouter", "gemini", "ollama", "nvidia", "qwencloud", "xiaomi", "mistral"]
# Provider configurations — keys loaded from file or .env
AI_KEYS_FILE = Path("data/api_keys.json")
@@ -61,6 +61,30 @@ def _load_provider_keys():
"model": os.getenv("OLLAMA_MODEL", "qwen2.5-coder:1.5b"),
"auth_header": "Bearer {api_key}",
},
"nvidia": {
"api_key": get_ai_key("NVIDIA_API_KEY"),
"base_url": "https://integrate.api.nvidia.com/v1",
"model": os.getenv("NVIDIA_MODEL", "meta/llama-3.1-405b-instruct"),
"auth_header": "Bearer {api_key}",
},
"qwencloud": {
"api_key": get_ai_key("QWENCLOUD_API_KEY"),
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
"model": os.getenv("QWENCLOUD_MODEL", "qwen-max"),
"auth_header": "Bearer {api_key}",
},
"xiaomi": {
"api_key": get_ai_key("XIAOMI_API_KEY"),
"base_url": "https://api.xiaomi.com/v1",
"model": os.getenv("XIAOMI_MODEL", "mimo-v2-pro"),
"auth_header": "Bearer {api_key}",
},
"mistral": {
"api_key": get_ai_key("MISTRAL_API_KEY"),
"base_url": "https://api.mistral.ai/v1",
"model": os.getenv("MISTRAL_MODEL", "mistral-large-latest"),
"auth_header": "Bearer {api_key}",
},
}
PROVIDERS = _load_provider_keys()
@@ -139,6 +163,7 @@ async def ai_complete(prompt: str, provider: ProviderName | None = None) -> str:
cfg = _get_provider_config(provider)
if cfg["name"] == "gemini":
return await _call_gemini(prompt, "You are a helpful assistant.")
# All other providers use OpenAI-compatible format
return await _call_deepseek_openrouter(prompt, "You are a helpful assistant.", provider)
+4
View File
@@ -42,6 +42,10 @@ async def api_status():
"deepseek": "DEEPSEEK_API_KEY",
"openrouter": "OPENROUTER_API_KEY",
"gemini": "GEMINI_API_KEY",
"nvidia": "NVIDIA_API_KEY",
"qwencloud": "QWENCLOUD_API_KEY",
"xiaomi": "XIAOMI_API_KEY",
"mistral": "MISTRAL_API_KEY",
}
providers = {}
for name, env_var in provider_keys.items():
+263
View File
@@ -0,0 +1,263 @@
"""BooksLM — Context collection and caching for directory-scoped AI chat.
Collects markdown files from an Obsidian vault directory, applies secret
redaction, builds a system prompt with file contents, and caches results
for repeated queries.
"""
import hashlib
import json
import logging
import os
import time
from pathlib import Path
from typing import Any
from backend.secret_redactor import redact_file_content
logger = logging.getLogger("obsigate.bookslm")
# ── Configuration limits ──
BOOKSLM_MAX_FILES = int(os.getenv("BOOKSLM_MAX_FILES", "200"))
BOOKSLM_MAX_TOTAL_CHARS = int(os.getenv("BOOKSLM_MAX_TOTAL_CHARS", "200000"))
BOOKSLM_MAX_FILE_CHARS = int(os.getenv("BOOKSLM_MAX_FILE_CHARS", "30000"))
# ── Cache ──
_cache: dict[str, dict[str, Any]] = {}
_CACHE_TTL = 300 # seconds
def _cache_key(vault_path: Path, directory: str, file_mtimes: list[tuple[str, float]]) -> str:
"""Build a SHA-256 cache key from vault+directory+file modification times."""
raw = json.dumps({
"vault": str(vault_path),
"dir": directory,
"mtimes": sorted(file_mtimes),
}, sort_keys=True)
return hashlib.sha256(raw.encode()).hexdigest()
def _should_skip(name: str) -> bool:
"""Return True if this file/directory name should be skipped."""
skip_prefixes = (".",)
skip_names = {"_attachments", "node_modules", ".git", ".obsidian", "__pycache__"}
if name in skip_names:
return True
if any(name.startswith(p) for p in skip_prefixes):
return True
return False
def _file_priority(path: Path) -> tuple[int, float]:
"""Sort key: README/index first, then by modification time descending.
Returns (priority_group, -mtime) so that:
- Group 0: README* and index* files (come first)
- Group 1: all other files (come after)
Within each group, newer files come first.
"""
name_lower = path.stem.lower()
if name_lower.startswith("readme") or name_lower.startswith("index"):
group = 0
else:
group = 1
try:
mtime = path.stat().st_mtime
except OSError:
mtime = 0.0
return (group, -mtime)
def collect_directory_context(vault_path: Path, directory: str) -> dict[str, Any]:
"""Walk a directory recursively, collect .md files with content.
Args:
vault_path: Absolute path to the vault root.
directory: Relative directory path within the vault (empty = root).
Returns:
Dict with keys: files, total_chars, file_count, directory_tree.
"""
target_dir = (vault_path / directory).resolve() if directory else vault_path.resolve()
vault_resolved = vault_path.resolve()
# Safety: ensure target is within vault
try:
target_dir.relative_to(vault_resolved)
except ValueError:
logger.warning(f"Directory outside vault: {target_dir}")
return {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
if not target_dir.exists() or not target_dir.is_dir():
return {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
# Check cache
file_mtimes: list[tuple[str, float]] = []
md_files: list[Path] = []
try:
for p in target_dir.rglob("*"):
# Skip hidden dirs/files and special dirs
parts = p.relative_to(target_dir).parts
if any(_should_skip(part) for part in parts):
continue
if p.is_file() and p.suffix.lower() == ".md":
md_files.append(p)
try:
file_mtimes.append((str(p.relative_to(target_dir)), p.stat().st_mtime))
except OSError:
file_mtimes.append((str(p.relative_to(target_dir)), 0.0))
except PermissionError:
logger.warning(f"Permission denied scanning {target_dir}")
return {"files": [], "total_chars": 0, "file_count": 0, "directory_tree": ""}
key = _cache_key(vault_resolved, directory, file_mtimes)
if key in _cache:
cached = _cache[key]
if time.time() - cached.get("_ts", 0) < _CACHE_TTL:
logger.debug(f"Cache hit for {directory}")
return {k: v for k, v in cached.items() if k != "_ts"}
# Sort by priority: README/index first, then by mtime descending
md_files.sort(key=_file_priority)
# Apply limits
collected: list[dict[str, Any]] = []
total_chars = 0
for p in md_files:
if len(collected) >= BOOKSLM_MAX_FILES:
break
if total_chars >= BOOKSLM_MAX_TOTAL_CHARS:
break
rel_path = str(p.relative_to(vault_resolved)).replace("\\", "/")
try:
content = p.read_text(encoding="utf-8", errors="replace")
except Exception as e:
logger.warning(f"Cannot read {rel_path}: {e}")
continue
# Redact secrets
content = redact_file_content(content, rel_path)
# Truncate if too long
if len(content) > BOOKSLM_MAX_FILE_CHARS:
content = content[:BOOKSLM_MAX_FILE_CHARS] + "\n\n[... tronqué]"
remaining = BOOKSLM_MAX_TOTAL_CHARS - total_chars
if len(content) > remaining:
content = content[:remaining] + "\n\n[... tronqué]"
title = p.stem.replace("-", " ").replace("_", " ").title()
file_type = "markdown"
collected.append({
"path": rel_path,
"title": title,
"content": content,
"type": file_type,
})
total_chars += len(content)
# Build directory tree
dir_tree = _build_directory_tree(target_dir, vault_resolved)
result = {
"files": collected,
"total_chars": total_chars,
"file_count": len(collected),
"directory_tree": dir_tree,
}
# Store in cache
_cache[key] = {**result, "_ts": time.time()}
logger.info(f"Collected {len(collected)} files ({total_chars} chars) from {directory or '/'}")
return result
def _build_directory_tree(target_dir: Path, vault_root: Path) -> str:
"""Build a text representation of the directory tree (dirs + .md files)."""
lines: list[str] = []
try:
for p in sorted(target_dir.rglob("*")):
parts = p.relative_to(target_dir).parts
if any(_should_skip(part) for part in parts):
continue
if p.is_dir():
depth = len(p.relative_to(target_dir).parts)
lines.append(f"{' ' * depth}{p.name}/")
elif p.is_file() and p.suffix.lower() == ".md":
depth = len(p.relative_to(target_dir).parts)
lines.append(f"{' ' * depth}{p.name}")
except PermissionError:
pass
return "\n".join(lines)
def build_system_prompt(context: dict[str, Any]) -> str:
"""Build a system prompt for directory-scoped AI chat.
Args:
context: Output of collect_directory_context().
Returns:
System prompt string with file contents.
"""
files = context.get("files", [])
file_count = context.get("file_count", 0)
total_chars = context.get("total_chars", 0)
# Rough token estimate (1 token ≈ 4 chars)
est_tokens = total_chars // 4
token_warning = ""
if est_tokens > 100_000:
token_warning = f"\n⚠️ Attention : le contexte est très volumineux (~{est_tokens:,} tokens estimés). Les réponses peuvent être moins précises.\n"
prompt = (
"Tu es un assistant de recherche documentaire. "
"Tu réponds UNIQUEMENT en te basant sur les documents fournis ci-dessous. "
"Cite tes sources avec le nom du fichier quand tu utilises une information. "
"Si l'information ne se trouve pas dans les documents, dis-le clairement."
f"\n\n📚 Contexte : {file_count} fichier(s) ({total_chars:,} caractères)"
f"{token_warning}\n"
)
# Directory tree
tree = context.get("directory_tree", "")
if tree:
prompt += f"\n📂 Arborescence du dossier :\n```\n{tree}\n```\n"
# File contents
prompt += "\n---\n"
for f in files:
prompt += f"\n## 📄 {f['title']} (`{f['path']}`)\n\n{f['content']}\n\n---\n"
prompt += "\nFin du contexte. Réponds à la question de l'utilisateur en te basant uniquement sur ces documents."
return prompt
def invalidate_cache(vault_path: Path | None = None, directory: str | None = None) -> int:
"""Invalidate cache entries.
Args:
vault_path: If provided, only invalidate entries for this vault.
directory: If provided, only invalidate entries for this directory.
Returns:
Number of cache entries removed.
"""
if vault_path is None and directory is None:
count = len(_cache)
_cache.clear()
return count
to_remove = []
for key, val in _cache.items():
# We can't easily reverse the hash, so clear everything if vault_path is given
# For targeted invalidation, callers should use directory
to_remove.append(key)
for k in to_remove:
del _cache[k]
return len(to_remove)
+147
View File
@@ -0,0 +1,147 @@
"""BooksLM API routes — directory-scoped AI chat for Obsidian vaults."""
import json
import logging
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from backend.auth.middleware import check_vault_access, require_auth
from backend.bookslm import build_system_prompt, collect_directory_context
from backend.indexer import get_vault_data
logger = logging.getLogger("obsigate.bookslm_routes")
router = APIRouter(prefix="/api/ai/bookslm", tags=["BooksLM"])
# ── Request models ──
class BooksLMContextRequest(BaseModel):
vault: str = Field(description="Vault name")
directory: str = Field(default="", description="Relative directory path within the vault")
class BooksLMChatRequest(BaseModel):
vault: str = Field(description="Vault name")
directory: str = Field(default="", description="Relative directory path within the vault")
message: str = Field(description="User message")
conversation_history: list[dict[str, str]] = Field(
default_factory=list,
description="Previous conversation turns [{role, content}]",
)
# ── Endpoints ──
@router.post("/context")
async def api_bookslm_context(
req: BooksLMContextRequest,
current_user=Depends(require_auth),
):
"""Collect directory context for BooksLM.
Returns file list, content, and metadata for the specified directory.
"""
if not check_vault_access(req.vault, current_user):
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{req.vault}'")
vault_data = get_vault_data(req.vault)
if not vault_data:
raise HTTPException(status_code=404, detail=f"Vault '{req.vault}' not found")
vault_path = Path(vault_data["path"])
context = collect_directory_context(vault_path, req.directory)
return context
@router.post("/chat")
async def api_bookslm_chat(
req: BooksLMChatRequest,
current_user=Depends(require_auth),
):
"""Chat with AI about directory contents (BooksLM).
Builds context from the directory, then sends the user message
with a system prompt containing all file contents to the AI provider.
Returns an SSE stream with the response.
"""
if not check_vault_access(req.vault, current_user):
raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{req.vault}'")
vault_data = get_vault_data(req.vault)
if not vault_data:
raise HTTPException(status_code=404, detail=f"Vault '{req.vault}' not found")
vault_path = Path(vault_data["path"])
# Collect context
context = collect_directory_context(vault_path, req.directory)
if context["file_count"] == 0:
raise HTTPException(status_code=404, detail="Aucun fichier markdown trouvé dans ce dossier")
# Build system prompt
system_prompt = build_system_prompt(context)
# Call AI provider
from backend.ai import DEFAULT_PROVIDER, PROVIDERS, _call_deepseek_openrouter, _call_gemini
# Build messages with conversation history
messages_text = ""
if req.conversation_history:
for turn in req.conversation_history:
role = turn.get("role", "user")
content = turn.get("content", "")
if role == "user":
messages_text += f"\n\nUtilisateur : {content}"
elif role == "assistant":
messages_text += f"\n\nAssistant : {content}"
# Current message
user_prompt = req.message
if messages_text:
user_prompt = f"Historique de la conversation :{messages_text}\n\nQuestion actuelle : {req.message}"
async def generate_sse():
try:
cfg_name = DEFAULT_PROVIDER
# Check if default provider is available
if cfg_name != "gemini" and cfg_name in PROVIDERS:
if not PROVIDERS[cfg_name].get("api_key"):
# Find first available provider
for pname, pcfg in PROVIDERS.items():
if pcfg.get("api_key") and pname != "gemini":
cfg_name = pname
break
if cfg_name == "gemini" and PROVIDERS.get("gemini", {}).get("api_key"):
response = await _call_gemini(user_prompt, system_prompt, temperature=0.3, max_tokens=4096)
else:
response = await _call_deepseek_openrouter(
user_prompt, system_prompt,
provider=cfg_name if cfg_name in PROVIDERS else None,
temperature=0.3,
max_tokens=4096,
)
# Send the full response as a single SSE event
data = json.dumps({"token": response}, ensure_ascii=False)
yield f"event: message\ndata: {data}\n\n"
yield "event: done\ndata: {}\n\n"
except Exception as e:
logger.error(f"BooksLM chat error: {e}")
error_data = json.dumps({"error": str(e)}, ensure_ascii=False)
yield f"event: error\ndata: {error_data}\n\n"
return StreamingResponse(
generate_sse(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
+19 -4
View File
@@ -695,6 +695,7 @@ except Exception: # pragma: no cover - WeasyPrint/GTK missing
# Multi-format export (HTML / MD bundle / ePub) — pure Python, no heavy deps.
from backend.export import export_epub, export_html, export_md_bundle, ExportError # noqa: E402
from backend.ai_routes import router as ai_router
from backend.bookslm_routes import router as bookslm_router
from backend.saved_searches import delete_saved, get_saved, save_search
from backend.share import (
create_share,
@@ -714,6 +715,7 @@ from backend.webhooks import (
app.include_router(auth_router)
app.include_router(ai_router)
app.include_router(bookslm_router)
# Resolve frontend path relative to this file
FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend"
@@ -3993,7 +3995,7 @@ async def api_get_ai_keys(current_user=Depends(require_admin)):
"""Return stored AI keys (values masked)."""
keys = _read_ai_keys()
masked = {}
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY"]:
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]:
val = keys.get(k, "") or os.environ.get(k, "")
if val:
masked[k] = val[:4] + "..." + val[-4:] if len(val) > 8 else "***"
@@ -4005,7 +4007,7 @@ async def api_get_ai_keys(current_user=Depends(require_admin)):
async def api_set_ai_keys(body: dict = Body(...), current_user=Depends(require_admin)):
"""Save AI keys. Pass {"DEEPSEEK_API_KEY":"sk-...","OPENROUTER_API_KEY":"...","GEMINI_API_KEY":"..."}"""
keys = _read_ai_keys()
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY"]:
for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]:
if body.get(k):
keys[k] = body[k]
_write_ai_keys(keys)
@@ -4020,6 +4022,10 @@ async def api_test_ai_keys(current_user=Depends(require_admin)):
("DEEPSEEK_API_KEY", "deepseek", "https://api.deepseek.com/v1/models", "Authorization"),
("OPENROUTER_API_KEY", "openrouter", "https://openrouter.ai/api/v1/models", "Authorization"),
("GEMINI_API_KEY", "gemini", "https://generativelanguage.googleapis.com/v1beta/models?key={key}", None),
("NVIDIA_API_KEY", "nvidia", "https://integrate.api.nvidia.com/v1/models", "Authorization"),
("QWENCLOUD_API_KEY", "qwencloud", "https://dashscope.aliyuncs.com/compatible-mode/v1/models", "Authorization"),
("XIAOMI_API_KEY", "xiaomi", "https://api.xiaomi.com/v1/models", "Authorization"),
("MISTRAL_API_KEY", "mistral", "https://api.mistral.ai/v1/models", "Authorization"),
]:
key = get_ai_key(key_name)
if not key:
@@ -4047,7 +4053,8 @@ async def api_list_ai_models(provider: str = Query(...), current_user=Depends(re
"""List available models for a given AI provider."""
provider = provider.lower()
if provider not in ("deepseek", "openrouter", "gemini"):
all_providers = ("deepseek", "openrouter", "gemini", "nvidia", "qwencloud", "xiaomi", "mistral")
if provider not in all_providers:
return {"models": [], "error": f"Unknown provider: {provider}"}
key_name = f"{provider.upper()}_API_KEY"
@@ -4060,8 +4067,16 @@ async def api_list_ai_models(provider: str = Query(...), current_user=Depends(re
url = f"https://generativelanguage.googleapis.com/v1beta/models?key={key}"
elif provider == "deepseek":
url = "https://api.deepseek.com/v1/models"
else: # openrouter
elif provider == "openrouter":
url = "https://openrouter.ai/api/v1/models"
elif provider == "nvidia":
url = "https://integrate.api.nvidia.com/v1/models"
elif provider == "qwencloud":
url = "https://dashscope.aliyuncs.com/compatible-mode/v1/models"
elif provider == "xiaomi":
url = "https://api.xiaomi.com/v1/models"
elif provider == "mistral":
url = "https://api.mistral.ai/v1/models"
try:
if provider == "gemini":