import asyncio import html as html_mod import json as _json import logging import os import re import secrets import shutil import string import urllib.request from contextlib import asynccontextmanager from pathlib import Path import frontmatter import mistune from fastapi import Body, Depends, FastAPI, HTTPException, Query, Request, WebSocket from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field from starlette.middleware.base import BaseHTTPMiddleware from backend.collab import authenticate_websocket, collab_manager from backend.history import ( get_bookmarks, toggle_bookmark, ) from backend.image_processor import preprocess_images from backend.indexer import ( add_vault_to_index, build_index, find_file_in_index, get_conflicts, get_vault_data, handle_file_move, index, remove_single_file, remove_vault_from_index, update_single_file, ) from backend.media_types import IMAGE_EXTENSIONS from backend.openapi_docs import ( API_DESCRIPTION, TAGS_METADATA, enrich_openapi_schema, render_api_landing, ) from backend.schemas import ( AIKeyDeleteResponse, AIKeysResponse, AIModelsResponse, AITestResponse, AppConfigResponse, BookmarksResponse, BookmarkToggleResponse, ConflictResolveResponse, ConflictsResponse, DashboardResponse, DiagnosticsResponse, RecentResponse, SavedSearch, StatusResponse, VaultActionResponse, VaultsStatusResponse, VaultStatsResponse, ) from backend.search import ( init_inverted_index, ) from backend.semantic_search import init_semantic_index from backend.services.backups import get_backup_dir as service_get_backup_dir from backend.services.errors import ServiceError from backend.services.recent import humanize_mtime, list_recent from backend.services.sanitizer import sanitize_html from backend.services.vaults import ( list_accessible_vaults, ) logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", ) logger = logging.getLogger("obsigate") # --------------------------------------------------------------------------- # Pydantic response models # --------------------------------------------------------------------------- class VaultInfo(BaseModel): """Summary information about a configured vault.""" name: str = Field(description="Display name of the vault") file_count: int = Field(description="Number of indexed files") tag_count: int = Field(description="Number of unique tags") type: str = Field(default="VAULT", description="Type of the vault mapping (VAULT or DIR)") # Filesystem mutation + search / suggest / graph models : voir backend.schemas (#85 T5, T6b) class BackupEntry(BaseModel): """A single backup version of a file.""" timestamp: int = Field(description="Unix timestamp of when the backup was created") datetime: str = Field(description="ISO 8601 datetime string") size: int = Field(description="File size in bytes") filename: str = Field(description="Backup filename on disk") class BackupListResponse(BaseModel): """Response listing all available backups for a file.""" vault: str = Field(description="Vault name") path: str = Field(description="Relative file path") backups: list[BackupEntry] = Field(description="Available backups, newest first") class DiffRequest(BaseModel): """Request parameters for generating a diff.""" version: int = Field(description="Timestamp of the backup version to compare") compare_with: int | None = Field(default=None, description="Timestamp of another backup version. If omitted, compares with the current file.") # --------------------------------------------------------------------------- # SSE Manager — voir backend.sse (ROADMAP #85 T4, instance partagée) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- from backend.search_executor import ( get_search_executor, init_search_executor, shutdown_search_executor, ) from backend.sse import sse_manager # --------------------------------------------------------------------------- # Application lifespan (replaces deprecated on_event) # --------------------------------------------------------------------------- from backend.watcher import VaultWatcher # File watcher handle (the search thread pool lives in backend.search_executor). _vault_watcher: VaultWatcher | None = None async def _on_vault_change(events: list): """Callback invoked by VaultWatcher when files change in watched vaults. Processes each event (create/modify/delete/move) and updates the index incrementally, then broadcasts SSE notifications. """ updated_vaults = set() changes = [] for event in events: vault_name = event["vault"] event_type = event["type"] src = event["src"] dest = event.get("dest") try: if event_type in ("created", "modified"): result = await update_single_file(vault_name, src) if result: changes.append({"action": "updated", "vault": vault_name, "path": result["path"]}) updated_vaults.add(vault_name) elif event_type == "deleted": result = await remove_single_file(vault_name, src) if result: changes.append({"action": "deleted", "vault": vault_name, "path": result["path"]}) updated_vaults.add(vault_name) elif event_type == "moved": result = await handle_file_move(vault_name, src, dest) if result: changes.append({"action": "moved", "vault": vault_name, "path": result["path"]}) updated_vaults.add(vault_name) except Exception as e: logger.error(f"Error processing {event_type} event for {src}: {e}") if changes: await sse_manager.broadcast("index_updated", { "vaults": list(updated_vaults), "changes": changes, "total_changes": len(changes), }) logger.info(f"Hot-reload: {len(changes)} change(s) in {list(updated_vaults)}") # --------------------------------------------------------------------------- # Authentication bootstrap # --------------------------------------------------------------------------- def bootstrap_admin(): """Create the initial admin account if no users exist. Reads OBSIGATE_ADMIN_USER and OBSIGATE_ADMIN_PASSWORD from environment. If no password is set, generates a random one and logs it ONCE. Only runs when auth is enabled and no users.json exists yet. """ from backend.auth.middleware import is_auth_enabled from backend.auth.user_store import create_user, has_users if not is_auth_enabled(): return if has_users(): return # Users already exist, skip admin_user = os.environ.get("OBSIGATE_ADMIN_USER", "admin") admin_pass = os.environ.get("OBSIGATE_ADMIN_PASSWORD", "") if not admin_pass: # Generate a random password and display it ONCE in logs admin_pass = "".join( secrets.choice(string.ascii_letters + string.digits) for _ in range(16) ) logger.warning("=" * 60) logger.warning("PREMIER DÉMARRAGE — Compte admin créé automatiquement") logger.warning(f" Utilisateur : {admin_user}") logger.warning(f" Mot de passe : {admin_pass}") logger.warning("CHANGEZ CE MOT DE PASSE dès la première connexion !") logger.warning("=" * 60) try: create_user(admin_user, admin_pass, role="admin", vaults=["*"]) logger.info(f"Admin '{admin_user}' créé avec succès") except PermissionError as e: logger.critical("=" * 60) logger.critical("DÉMARRAGE IMPOSSIBLE : Erreur de permission sur le dossier 'data'") logger.critical("L'indexation et l'authentification ne peuvent pas fonctionner.") logger.critical("FIX : Vérifiez les droits du volume /app/data sur l'hôte.") logger.critical("Exemple : sudo chown -R 1000:1000 /DOCKER_CONFIG/ObsiGate/data") logger.critical("=" * 60) raise e # --------------------------------------------------------------------------- # Security headers middleware # --------------------------------------------------------------------------- class SecurityHeadersMiddleware(BaseHTTPMiddleware): """Add security headers to all HTTP responses.""" async def dispatch(self, request, call_next): response = await call_next(request) response.headers["X-Content-Type-Options"] = "nosniff" response.headers["X-Frame-Options"] = "SAMEORIGIN" response.headers["X-XSS-Protection"] = "1; mode=block" response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" # A route may set a stricter per-response policy (e.g. ``sandbox`` for # standalone SVG, #108-B3); keep it instead of overwriting it. if "Content-Security-Policy" not in response.headers: response.headers["Content-Security-Policy"] = ( "default-src 'self'; " "script-src 'self' 'unsafe-inline' blob: https://cdnjs.cloudflare.com https://unpkg.com https://esm.sh https://cdn.jsdelivr.net https://static.cloudflareinsights.com; " "style-src 'self' 'unsafe-inline' https://cdnjs.cloudflare.com https://fonts.googleapis.com https://cdn.jsdelivr.net https://esm.sh; " "img-src 'self' data: blob:; " "connect-src 'self' blob: https://esm.sh https://unpkg.com https://cdnjs.cloudflare.com https://fonts.googleapis.com https://fonts.gstatic.com https://cdn.jsdelivr.net; " "font-src 'self' data: https://fonts.gstatic.com https://esm.sh; " "worker-src 'self' blob:; " "frame-src 'self' blob:; " "object-src 'none'; " "base-uri 'self'; " "form-action 'self'; " "frame-ancestors 'self';" ) # Static assets are NOT content-hashed, so they must revalidate: # ``immutable``/long max-age made Cloudflare and mobile browsers serve # a stale build for a year (the service worker cache compounded it). # ``no-cache`` keeps caching but forces revalidation (ETag/Last-Modified). if request.url.path.startswith("/static/"): response.headers["Cache-Control"] = "no-cache" return response def _guard_insecure_auth() -> None: """Warn or refuse to start when authentication is disabled (BUG-037). With ``OBSIGATE_AUTH_ENABLED=false`` every request is served as an anonymous admin. That is convenient for local use but dangerous when the process is reachable from a network. Binding to a non-loopback host without the explicit ``OBSIGATE_ALLOW_INSECURE=true`` opt-in is refused. """ from backend.auth.middleware import ( bind_host_from_argv, is_auth_enabled, is_insecure_mode_allowed, is_loopback_host, ) if is_auth_enabled(): return if is_insecure_mode_allowed(): logger.warning( "Authentication is DISABLED and OBSIGATE_ALLOW_INSECURE=true: every request " "is treated as an anonymous administrator. Do not expose this instance." ) return host = bind_host_from_argv() if not is_loopback_host(host): raise RuntimeError( "Refusing to start: authentication is disabled (OBSIGATE_AUTH_ENABLED=false) " f"while binding to a non-loopback address ('{host}'). This would expose an " "unauthenticated instance with admin access. Enable authentication, or set " "OBSIGATE_ALLOW_INSECURE=true if you really know what you are doing." ) logger.warning( "Authentication is DISABLED (OBSIGATE_AUTH_ENABLED=false): every request is " "treated as an anonymous administrator. This is only safe on a trusted, " "loopback-only deployment." ) @asynccontextmanager async def lifespan(app: FastAPI): """Application lifespan: build index on startup, cleanup on shutdown.""" global _vault_watcher # Thread pool for offloading CPU-bound search from the event loop. # Sized to 2 workers so concurrent searches don't starve other requests. init_search_executor() # BUG-037: refuse to expose an unauthenticated instance on a public bind. _guard_insecure_auth() # Bootstrap admin account if needed bootstrap_admin() logger.info("ObsiGate starting — building index in background...") async def _progress_cb(event_type: str, data: dict): await sse_manager.broadcast("index_" + event_type, data) async def _background_startup(): logger.info("Background indexing started") await build_index(_progress_cb) # Build inverted index in a thread pool to avoid blocking the event loop. # The inverted index rebuild is CPU-bound (tokenization, indexing) and # would freeze HTTP responses if run in the async event loop. loop = asyncio.get_running_loop() await loop.run_in_executor(get_search_executor(), init_inverted_index) # Build the semantic (embedding) index in the same background thread pool. await loop.run_in_executor(get_search_executor(), init_semantic_index) # BUG-040: extract the PDF text deferred during the scan now that the # index and inverted index are queryable (keeps startup non-blocking). from backend.indexer import enrich_pdf_texts await enrich_pdf_texts() # Scan for plugins in all vaults logger.info("Scanning for plugins...") from backend.indexer import vault_config from backend.plugins import get_plugin_registry registry = get_plugin_registry() for vault_name, cfg in vault_config.items(): vault_path = cfg.get("path") if vault_path: try: plugins = registry.scan_vault(vault_name, vault_path) logger.info(f"Vault '{vault_name}': found {len(plugins)} plugin(s)") from backend.plugins import emit_vault_mounted emit_vault_mounted(vault_name, vault_path) except Exception as e: logger.warning(f"Plugin scan failed for vault '{vault_name}': {e}") # Start file watcher config = _load_config() watcher_enabled = config.get("watcher_enabled", True) if watcher_enabled: use_polling = config.get("watcher_use_polling", False) polling_interval = config.get("watcher_polling_interval", 5.0) debounce = config.get("watcher_debounce", 2.0) global _vault_watcher _vault_watcher = VaultWatcher( on_file_change=_on_vault_change, debounce_seconds=debounce, use_polling=use_polling, polling_interval=polling_interval, ) from backend.indexer import vault_config vaults_to_watch = {name: cfg["path"] for name, cfg in vault_config.items()} await _vault_watcher.start(vaults_to_watch) logger.info("File watcher started in background.") else: logger.info("File watcher disabled by configuration.") logger.info("Background startup complete.") asyncio.create_task(_background_startup()) logger.info("ObsiGate ready (listening for requests while indexing).") yield # Shutdown await collab_manager.stop() if _vault_watcher: await _vault_watcher.stop() _vault_watcher = None shutdown_search_executor() from backend.version import get_version app = FastAPI( title="ObsiGate API", version=get_version(), lifespan=lifespan, description=API_DESCRIPTION.strip(), openapi_tags=TAGS_METADATA, docs_url="/docs", redoc_url="/redoc", openapi_url="/openapi.json", contact={"name": "ObsiGate", "url": "https://git.dracodev.net/Projets/ObsiGate"}, license_info={"name": "MIT"}, ) # Enrich the auto-generated OpenAPI 3.1 schema (#72): tags per category, # examples, security schemes and documented error responses. _original_openapi = app.openapi def _custom_openapi(): if app.openapi_schema: return app.openapi_schema schema = _original_openapi() app.openapi_schema = enrich_openapi_schema(schema) return app.openapi_schema app.openapi = _custom_openapi # type: ignore[method-assign] @app.exception_handler(ServiceError) async def _service_error_handler(request: Request, exc: ServiceError): """Map shared-layer domain errors to HTTP responses (``{"detail": ...}``).""" return JSONResponse(status_code=exc.status, content={"detail": exc.message}) # GZip compression — reduces bandwidth by ~70% for text responses # Custom wrapper: skip compression for SSE streams (/api/events) from fastapi.middleware.gzip import GZipMiddleware from starlette.types import Receive, Scope, Send class SSESafeGZipMiddleware(GZipMiddleware): """GZip middleware that skips SSE (Server-Sent Events) streams. GZip buffering breaks incremental streaming required by SSE. We detect SSE endpoints by path and bypass compression entirely. """ # SSE endpoints that must not be buffered by GZip. _SSE_PATHS = ( "/api/events", "/api/admin/stream", "/api/ai/bookslm/chat", "/api/ai/bookslm/agent", "/mcp", ) async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] == "http" and scope.get("path") in self._SSE_PATHS: # Bypass GZip: passthrough directly to the inner app await self.app(scope, receive, send) else: await super().__call__(scope, receive, send) app.add_middleware(SSESafeGZipMiddleware, minimum_size=1000) # Security headers on all responses app.add_middleware(SecurityHeadersMiddleware) # Auth router # Multi-format export (HTML / MD bundle / ePub) — voir backend.routers.files_media (#85 T6c). from backend.ai_routes import router as ai_router from backend.audit import log_file_delete from backend.auth.middleware import ( check_vault_access, require_admin, require_auth, ) from backend.auth.router import router as auth_router from backend.bookslm_routes import router as bookslm_router from backend.routers.backups import router as backups_router from backend.routers.files_media import router as files_media_router from backend.routers.files_read import router as files_read_router from backend.routers.files_write import router as files_write_router from backend.routers.health import router as health_router from backend.routers.search import router as search_router from backend.routers.sharing import router as sharing_router from backend.routers.webhooks import router as webhooks_router from backend.saved_searches import delete_saved, get_saved, save_search from backend.secret_redactor import redact_file_content from backend.skills_routes import router as skills_router app.include_router(auth_router) app.include_router(ai_router) app.include_router(bookslm_router) app.include_router(skills_router) app.include_router(health_router) # ROADMAP #85 T1 — System / health app.include_router(search_router) # ROADMAP #85 T5 — Search app.include_router(backups_router) # ROADMAP #85 T4 — Backups app.include_router(files_read_router) # ROADMAP #85 T6a — Files read app.include_router(files_media_router) # ROADMAP #85 T6c — Media/export app.include_router(files_write_router) # ROADMAP #85 T6b — Files write app.include_router(webhooks_router) # ROADMAP #85 T2 — Webhooks app.include_router(sharing_router) # ROADMAP #85 T3 — Sharing # Admin Dashboard endpoints (system stats, audit logs, backups, stream) try: from backend.admin import router as admin_router app.include_router(admin_router) logger.info("Admin dashboard router mounted at /api/admin/*") except ImportError as e: logger.warning(f"Could not load admin dashboard router: {e}") # Push Notifications endpoints (Web Push API + VAPID) try: from backend.push import router as push_router app.include_router(push_router) logger.info("Push notifications router mounted at /api/push/*") except ImportError as e: logger.warning(f"Could not load push notifications router: {e}") # Plugins system endpoints try: from backend.plugins import router as plugins_router app.include_router(plugins_router) logger.info("Plugins router mounted at /api/plugins/*") except ImportError as e: logger.warning(f"Could not load plugins router: {e}") # MCP server (Streamable HTTP) for external clients (#79 phase E) try: from backend.mcp.server import McpMount, mcp_app app.router.routes.append(McpMount(mcp_app)) logger.info("MCP server mounted at /mcp") except Exception as e: # pragma: no cover - optional dependency logger.warning(f"Could not mount MCP server: {e}") # Resolve frontend path relative to this file FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend" # --------------------------------------------------------------------------- # API documentation landing page (#72) # --------------------------------------------------------------------------- @app.get("/api", include_in_schema=False, response_class=HTMLResponse) @app.get("/api/", include_in_schema=False, response_class=HTMLResponse) async def api_docs_landing(): """Human-friendly API documentation landing page (links to /docs, /redoc).""" return HTMLResponse(render_api_landing(get_version())) # --------------------------------------------------------------------------- # Path safety helper : voir backend.routers.helpers (#85 T6a, T6c) # --------------------------------------------------------------------------- def _resolve_safe_path(vault_root: Path, relative_path: str | None) -> Path: """Resolve a relative path safely within the vault root. Thin wrapper around the shared :func:`backend.services.paths.resolve_safe_path` (single implementation used by both routes and tools). The raised :class:`ServiceError` is mapped to an ``HTTPException`` response by the global exception handler in this module. Args: vault_root: The vault's root directory (absolute). relative_path: The user-supplied relative path. Returns: Resolved absolute ``Path``. """ from backend.services.paths import resolve_safe_path as _service_resolve return _service_resolve(vault_root, relative_path) def _backup_file(file_path: Path, vault_name: str, relative_path: str): """Create a timestamped backup of a file before modification. Thin wrapper around :func:`backend.services.backups.create_backup` (single implementation used by both routes and tools). Backups are stored in ``{backup_root}/{vault}/{relative_path}.{timestamp}.bak``; the operation is best-effort and never blocks the caller. """ from backend.services.backups import create_backup create_backup(file_path, vault_name, relative_path) def _check_vault_writable(vault_root: Path) -> bool: """Check if a vault is writable (not mounted read-only). Args: vault_root: The vault's root directory (absolute). Returns: True if the vault is writable, False otherwise. """ return os.access(vault_root, os.W_OK) # --------------------------------------------------------------------------- # Markdown rendering helpers (singleton renderer) # --------------------------------------------------------------------------- import unicodedata def _heading_slugify(text: str) -> str: """Generate a URL-safe slug from heading text. Matches the JavaScript slugify algorithm exactly using Unicode-aware character classification: 1. Strip HTML tags (e.g. wikilink spans rendered inside headings) 2. Decode HTML entities (e.g. ``&`` → ``&``) 3. Lowercase 4. NFD normalize + strip combining marks 5. Keep only Unicode letters, numbers, spaces, hyphens 6. Replace spaces with hyphens, collapse multiple hyphens Args: text: The heading text content (may contain inline HTML). Returns: A URL-safe slug string. """ # Strip any inline HTML so it does not pollute the slug text = re.sub(r"<[^>]+>", "", text) # Decode HTML entities so & becomes & before slugification text = html_mod.unescape(text) text = text.lower() text = unicodedata.normalize("NFD", text) text = "".join(ch for ch in text if not unicodedata.combining(ch)) # Unicode-aware: keep letters (L*), numbers (N*), spaces, and hyphens cleaned = [] for ch in text: cat = unicodedata.category(ch) if cat.startswith('L') or cat.startswith('N') or ch in (' ', '-'): cleaned.append(ch) text = "".join(cleaned) text = re.sub(r"\s+", "-", text) text = re.sub(r"-+", "-", text) result = text.strip("-") return result if result else "heading" def _add_heading_ids(html: str) -> str: """Post-process rendered HTML to add IDs to heading tags. Adds an ``id`` attribute to every ``

`` through ``

`` tag using a slug generated from the heading's text content. Duplicate slugs get a ``-2``, ``-3``, etc. suffix. Args: html: Rendered HTML string. Returns: HTML with heading IDs injected. """ used_ids: dict[str, int] = {} def _replace_heading(match): tag = match.group(1) content = match.group(2) slug = _heading_slugify(content) count = used_ids.get(slug, 0) used_ids[slug] = count + 1 if count > 0: slug = f"{slug}-{count + 1}" return f'<{tag} id="{slug}">{content}' # Match h1-h6 tags with text content (no existing id attribute) return re.sub( r'<(h[1-6])>([^<]*(?:<(?!/?h[1-6])[^<]*)*)', _replace_heading, html, ) # Cached mistune renderer — avoids re-creating on every request _markdown_renderer = mistune.create_markdown( escape=False, plugins=["table", "strikethrough", "footnotes", "task_lists"], ) def _convert_wikilinks(content: str, current_vault: str) -> str: """Convert ``[[wikilinks]]`` and ``[[target|display]]`` to clickable HTML. Supports: - Internal file links: ``[[My Note]]`` / ``[[My Note|display]]`` - Same-document anchors: ``[[#Heading]]`` / ``[[#Heading|display]]`` Resolved file links get a ``data-vault`` / ``data-path`` attribute pair. Anchor links target the slugified heading ID in the current document. Unresolved links are rendered as ````. Args: content: Markdown string potentially containing wikilinks. current_vault: Active vault name for resolution priority. Returns: Markdown string with wikilinks replaced by HTML anchors. """ def _replace(match): target = match.group(1).strip() display = match.group(2).strip() if match.group(2) else target # Same-document anchor link: [[#Heading|display]] if target.startswith("#"): anchor_text = target[1:].strip() anchor_slug = _heading_slugify(anchor_text) link_display = display if display != target else anchor_text return f'{link_display}' found = find_file_in_index(target, current_vault) if found: return ( f'{display}' ) return f'{display}' pattern = r'\[\[([^\]|]+)(?:\|([^\]]+))?\]\]' return re.sub(pattern, _replace, content) def _normalize_line_breaks(text: str) -> str: """Convert single newlines to hard breaks (matching Obsidian default behavior). In standard Markdown, a single ``\\n`` is a "soft break" — it renders as a space, not a visible line break. Obsidian defaults to treating single newlines as hard breaks (equivalent to ``
``). This function pre-processes the Markdown source so that mistune renders standalone lines on separate rows, while still honouring blank lines as paragraph separators. Fenced code blocks (`` ``` ``) are left untouched so their internal newlines are preserved verbatim. """ parts = re.split(r"(```[\s\S]*?```)", text) for i, part in enumerate(parts): if part.startswith("```"): continue # Protect fenced code blocks # Single \n (not preceded or followed by another \n) → two spaces + \n parts[i] = re.sub(r"(? str: """Render a markdown string to HTML with wikilink and image support. Uses the cached singleton mistune renderer for performance. Args: raw_md: Raw markdown text (frontmatter already stripped). vault_name: Current vault for wikilink resolution context. current_file_path: Absolute path to the current markdown file. Returns: HTML string. """ # Get vault data for image resolution vault_data = get_vault_data(vault_name) vault_root = Path(vault_data["path"]) if vault_data else None attachments_path = vault_data.get("config", {}).get("attachmentsPath") if vault_data else None # Redact secrets before rendering (P0 security) raw_md = redact_file_content(raw_md, str(current_file_path) if current_file_path else "") # Preprocess images first if vault_root: raw_md = preprocess_images(raw_md, vault_name, vault_root, current_file_path, attachments_path) # Convert wikilinks converted = _convert_wikilinks(raw_md, vault_name) # Normalize line breaks to match Obsidian behavior (single \n → hard break) converted = _normalize_line_breaks(converted) rendered = _markdown_renderer(converted) # Add heading IDs for TOC navigation rendered = _add_heading_ids(rendered) # Sanitize: raw HTML in vault content must never reach the DOM (BUG-021). rendered = sanitize_html(rendered) return rendered # --------------------------------------------------------------------------- # API Endpoints — System / health : voir backend.routers.health (#85 T1) # --------------------------------------------------------------------------- @app.get("/api/vaults", response_model=list[VaultInfo]) async def api_vaults(current_user=Depends(require_auth)): """List configured vaults the user has access to. Returns: List of vault summary objects filtered by user permissions. """ return list_accessible_vaults(current_user) @app.get("/api/recent", response_model=RecentResponse) async def api_recent(limit: int | None = Query(None), vault: str | None = Query(None), mode: str | None = Query("opened"), current_user=Depends(require_auth)): config = _load_config() actual_limit = limit if limit is not None else config.get("recent_files_limit", 20) username = current_user.get("username") user_vaults = current_user.get("_token_vaults") or current_user.get("vaults", []) return list_recent( username, user_vaults, vault=vault, limit=actual_limit, mode=mode or "opened", ) @app.get("/api/bookmarks", response_model=BookmarksResponse) async def api_bookmarks(vault: str | None = Query(None), current_user=Depends(require_auth)): username = current_user.get("username") user_vaults = current_user.get("_token_vaults") or current_user.get("vaults", []) if not username: return {"files": []} history = get_bookmarks(username, vault_filter=vault) files_resp = [] for item in history: v_name = item["vault"] if "*" not in user_vaults and v_name not in user_vaults: continue # Find in index to get metadata f_idx = find_file_in_index(item["path"], v_name) if f_idx: files_resp.append({ "path": f_idx["path"], "title": f_idx.get("title") or item["path"].split("/")[-1], "vault": v_name, "mtime": item["bookmarked_at"], "mtime_human": humanize_mtime(item["bookmarked_at"]), "size_bytes": f_idx.get("size", 0), "tags": [f"#{t}" for t in f_idx.get("tags", [])][:5], "bookmarked": True }) else: files_resp.append({ "path": item["path"], "title": item.get("title") or item["path"].split("/")[-1], "vault": v_name, "mtime": item["bookmarked_at"], "mtime_human": humanize_mtime(item["bookmarked_at"]), "tags": [], "bookmarked": True }) return { "files": files_resp, "total": len(files_resp) } class BookmarkToggleRequest(BaseModel): vault: str path: str title: str | None = None @app.post("/api/bookmarks/toggle", response_model=BookmarkToggleResponse) async def api_toggle_bookmark(req: BookmarkToggleRequest, current_user=Depends(require_auth)): username = current_user.get("username") if not username: raise HTTPException(status_code=401, detail="Not authenticated") # Check vault access if not check_vault_access(req.vault, current_user): raise HTTPException(status_code=403, detail="Access denied to vault") is_now_bookmarked = toggle_bookmark(username, req.vault, req.path, req.title or "") # Update the file's YAML frontmatter: favoris: true/false vault_data = get_vault_data(req.vault) if vault_data: file_path = _resolve_safe_path(Path(vault_data["path"]), req.path) if file_path.exists() and file_path.suffix == ".md": try: raw = file_path.read_text(encoding="utf-8", errors="replace") post = frontmatter.loads(raw) if is_now_bookmarked: post.metadata["favoris"] = True elif "favoris" in post.metadata: del post.metadata["favoris"] new_raw = frontmatter.dumps(post) _backup_file(file_path, req.vault, req.path) file_path.write_text(new_raw, encoding="utf-8") await update_single_file(req.vault, str(file_path)) except Exception as e: logger.warning(f"Failed to update favoris metadata on {req.vault}/{req.path}: {e}") return {"bookmarked": is_now_bookmarked} @app.get("/api/saved-searches", response_model=list[SavedSearch]) async def api_saved_searches(current_user=Depends(require_auth)): username = current_user.get("username") if not username: raise HTTPException(401) return get_saved(username) @app.post("/api/saved-searches", response_model=SavedSearch) async def api_save_search(body: dict = Body(...), current_user=Depends(require_auth)): username = current_user.get("username") if not username: raise HTTPException(401) return save_search(username, body) @app.delete("/api/saved-searches/{search_id}", response_model=StatusResponse) async def api_delete_saved_search(search_id: str, current_user=Depends(require_auth)): username = current_user.get("username") if not username: raise HTTPException(401) if not delete_saved(username, search_id): raise HTTPException(404, "Not found") return {"status": "deleted"} # --------------------------------------------------------------------------- # File browse & read endpoints : voir backend.routers.files_read (#85 T6a) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # File PDF / export / guide / media endpoints : voir backend.routers.files_media (#85 T6c) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # File & directory mutations : voir backend.routers.files_write (#85 T6b) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # File creation and rename endpoints : voir backend.routers.files_write (#85 T6b) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # File creation and rename endpoints : voir backend.routers.files_write (#85 T6b) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Backup & Diff endpoints # --------------------------------------------------------------------------- def _get_backup_dir(vault_name: str, relative_path: str) -> Path: """Return the directory where backups for a specific file are stored. Thin wrapper around :func:`backend.services.backups.get_backup_dir` (single implementation used by both routes and tools). """ return service_get_backup_dir(vault_name, relative_path) # --------------------------------------------------------------------------- # File-level backup endpoints : voir backend.routers.backups (#85 T4) # --------------------------------------------------------------------------- # File backlinks + view endpoints : voir backend.routers.files_read (#85 T6a) # Range helper : voir backend.routers.helpers.stream_file_with_range (#85 T6c) # PDF stream/info endpoints : voir backend.routers.files_media (#85 T6c) # --------------------------------------------------------------------------- # Search / suggest / graph / index-reload : voir backend.routers.search (#85 T5) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # SSE endpoint — Server-Sent Events stream # --------------------------------------------------------------------------- @app.get( "/api/events", response_class=StreamingResponse, responses={200: {"content": {"text/event-stream": {}}, "description": "Server-Sent Events stream"}}, ) async def api_events(current_user=Depends(require_auth)): """SSE stream for real-time index update notifications. Sends keepalive comments every 30s. Events: - ``index_updated``: partial index change (file create/modify/delete/move) - ``index_reloaded``: full re-index completed - ``vault_added``: new vault added dynamically - ``vault_removed``: vault removed dynamically """ queue = await sse_manager.connect() async def event_generator(): try: # Send initial connection event yield f"event: connected\ndata: {_json.dumps({'sse_clients': sse_manager.client_count})}\n\n" while True: try: msg = await asyncio.wait_for(queue.get(), timeout=30.0) yield f"event: {msg['event']}\ndata: {msg['data']}\n\n" except asyncio.TimeoutError: # Keepalive comment yield ": keepalive\n\n" except asyncio.CancelledError: break finally: sse_manager.disconnect(queue) return StreamingResponse( event_generator(), media_type="text/event-stream", headers={ "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no", }, ) # --------------------------------------------------------------------------- # Dynamic vault management endpoints # --------------------------------------------------------------------------- @app.post("/api/vaults/add", response_model=VaultStatsResponse) async def api_add_vault(body: dict = Body(...), current_user=Depends(require_admin)): """Add a new vault dynamically without restarting. Body: name: Display name for the vault. path: Absolute filesystem path to the vault directory. """ name = body.get("name", "").strip() vault_path = body.get("path", "").strip() if not name or not vault_path: raise HTTPException(status_code=400, detail="Both 'name' and 'path' are required") if name in index: raise HTTPException(status_code=409, detail=f"Vault '{name}' already exists") if not Path(vault_path).exists(): raise HTTPException(status_code=400, detail=f"Path does not exist: {vault_path}") stats = await add_vault_to_index(name, vault_path) # Start watching the new vault if _vault_watcher: await _vault_watcher.add_vault(name, vault_path) await sse_manager.broadcast("vault_added", {"vault": name, "stats": stats}) return {"status": "ok", "vault": name, "stats": stats} @app.delete("/api/vaults/{vault_name}", response_model=VaultActionResponse) async def api_remove_vault(vault_name: str, current_user=Depends(require_admin)): """Remove a vault from the index and stop watching it. Args: vault_name: Name of the vault to remove. """ if vault_name not in index: raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found") # Stop watching if _vault_watcher: await _vault_watcher.remove_vault(vault_name) await remove_vault_from_index(vault_name) await sse_manager.broadcast("vault_removed", {"vault": vault_name}) return {"status": "ok", "vault": vault_name} @app.get("/api/vaults/status", response_model=VaultsStatusResponse) async def api_vaults_status(current_user=Depends(require_auth)): """Detailed status of all vaults including watcher state. Returns per-vault: file count, tag count, watching status, vault path. """ statuses = {} for vname, vdata in index.items(): watching = _vault_watcher is not None and vname in _vault_watcher.observers statuses[vname] = { "file_count": len(vdata.get("files", [])), "tag_count": len(vdata.get("tags", {})), "path": vdata.get("path", ""), "watching": watching, } return { "vaults": statuses, "watcher_active": _vault_watcher is not None, "sse_clients": sse_manager.client_count, } # --------------------------------------------------------------------------- # Image / media / attachments / vault-settings : voir backend.routers.files_media (#85 T6c) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Vault Settings API — Display preferences : voir backend.routers.files_media (#85 T6c) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Backup Management API # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Global backup endpoints : voir backend.routers.backups (#85 T4) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Configuration API # --------------------------------------------------------------------------- _BASE_DIR = Path(__file__).resolve().parent.parent _CONFIG_PATH = _BASE_DIR / "data" / "config.json" _DEFAULT_CONFIG = { "search_workers": 2, "debounce_ms": 300, "results_per_page": 50, "min_query_length": 2, "search_timeout_ms": 30000, "max_content_size": 100000, "snippet_context_chars": 120, "max_snippet_highlights": 5, "title_boost": 3.0, "path_boost": 1.5, "watcher_enabled": True, "watcher_use_polling": False, "watcher_polling_interval": 5.0, "watcher_debounce": 2.0, "tag_boost": 2.0, "prefix_max_expansions": 50, "recent_files_limit": 20, "max_backups_per_file": 10, "ai_default_provider": "deepseek", "ai_default_models": {}, } def _load_config() -> dict: """Load config from disk, merging with defaults.""" config = dict(_DEFAULT_CONFIG) if _CONFIG_PATH.exists(): try: stored = _json.loads(_CONFIG_PATH.read_text(encoding="utf-8")) config.update(stored) except Exception as e: logger.warning(f"Failed to read config.json: {e}") return config def _save_config(config: dict) -> None: """Persist config to disk.""" try: _CONFIG_PATH.write_text( _json.dumps(config, indent=2, ensure_ascii=False), encoding="utf-8", ) except Exception as e: logger.error(f"Failed to write config.json: {e}") raise HTTPException(status_code=500, detail=f"Failed to save config: {e}") @app.get("/api/config", response_model=AppConfigResponse) async def api_get_config(current_user=Depends(require_auth)): """Return current configuration with defaults for missing keys.""" return _load_config() @app.post("/api/config", response_model=AppConfigResponse) async def api_set_config(body: dict = Body(...), current_user=Depends(require_admin)): """Update configuration. Only known keys are accepted. Keys matching ``_DEFAULT_CONFIG`` are validated and persisted. Unknown keys are silently ignored. Returns the full merged config after update. """ current = _load_config() updated_keys = [] for key, value in body.items(): if key in _DEFAULT_CONFIG: expected_type = type(_DEFAULT_CONFIG[key]) if isinstance(value, expected_type) or (expected_type is float and isinstance(value, (int, float))): current[key] = value updated_keys.append(key) else: raise HTTPException( status_code=400, detail=f"Invalid type for '{key}': expected {expected_type.__name__}, got {type(value).__name__}", ) _save_config(current) if any(k.startswith("ai_") for k in updated_keys): try: from backend.ai import reload_ai_config reload_ai_config() except Exception as e: logger.warning(f"Failed to reload AI config: {e}") logger.info(f"Config updated: {updated_keys}") return current # --------------------------------------------------------------------------- # AI API Keys — stored in data/api_keys.json, fallback to .env # --------------------------------------------------------------------------- from backend.ai import PROVIDERS, _read_ai_keys, get_ai_key AI_KEYS_FILE = Path("data/api_keys.json") def _write_ai_keys(data: dict): AI_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True) tmp = AI_KEYS_FILE.with_suffix(".tmp") tmp.write_text(_json.dumps(data, indent=2), encoding="utf-8") tmp.replace(AI_KEYS_FILE) @app.get("/api/config/ai-keys", response_model=AIKeysResponse) async def api_get_ai_keys(current_user=Depends(require_admin)): """Return stored AI keys (values masked).""" keys = _read_ai_keys() masked = {} for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]: val = keys.get(k, "") or os.environ.get(k, "") if val: masked[k] = val[:4] + "..." + val[-4:] if len(val) > 8 else "***" else: masked[k] = "" return masked @app.post("/api/config/ai-keys", response_model=StatusResponse) async def api_set_ai_keys(body: dict = Body(...), current_user=Depends(require_admin)): """Save AI keys. Pass {"DEEPSEEK_API_KEY":"sk-...","OPENROUTER_API_KEY":"...","GEMINI_API_KEY":"..."}""" keys = _read_ai_keys() for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]: if body.get(k): keys[k] = body[k] _write_ai_keys(keys) logger.info("AI keys updated") return {"status": "ok"} @app.delete("/api/config/ai-keys/{provider_env}", response_model=AIKeyDeleteResponse) async def api_delete_ai_key(provider_env: str, current_user=Depends(require_admin)): """Delete a specific AI provider key from storage.""" allowed = {"DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"} key_name = provider_env.upper() if key_name not in allowed: raise HTTPException(status_code=400, detail=f"Clé inconnue: {provider_env}") keys = _read_ai_keys() if key_name in keys: del keys[key_name] _write_ai_keys(keys) # Also clear from env at runtime so get_ai_key() no longer finds it os.environ.pop(key_name, None) logger.info(f"AI key deleted: {key_name}") return {"status": "deleted", "key": key_name} # --------------------------------------------------------------------------- # Tool & connected-source keys (#103) — same store as the AI provider keys # --------------------------------------------------------------------------- from backend.tools.secrets import ( TOOL_KEY_NAMES as _TOOL_KEY_NAMES, ) from backend.tools.secrets import ( delete_tool_key as _delete_tool_key, ) from backend.tools.secrets import ( get_tool_key as _get_tool_key, ) from backend.tools.secrets import ( mask_value as _mask_tool_value, ) from backend.tools.secrets import ( set_tool_key as _set_tool_key, ) @app.get("/api/config/tool-keys", response_model=AIKeysResponse) async def api_get_tool_keys(current_user=Depends(require_admin)): """Return tool/connected-source configuration (tokens masked, URLs clear).""" masked = {} for name in _TOOL_KEY_NAMES: masked[name] = _mask_tool_value(name, _get_tool_key(name)) return masked @app.post("/api/config/tool-keys", response_model=StatusResponse) async def api_set_tool_keys(body: dict = Body(...), current_user=Depends(require_admin)): """Save tool/connected-source keys. Only whitelisted names (``backend.tools.secrets.TOOL_KEY_NAMES``) are accepted: Tavily/Brave/SerpAPI/Exa API keys, Gitea URL + token, GitHub token. Empty values delete the stored entry. """ updated = [] for name, value in body.items(): if name not in _TOOL_KEY_NAMES: raise HTTPException(status_code=400, detail=f"Clé inconnue: {name}") if value is not None and not isinstance(value, str): raise HTTPException(status_code=400, detail=f"Type invalide pour {name}") _set_tool_key(name, value or "") updated.append(name) logger.info(f"Tool keys updated: {updated}") return {"status": "ok"} @app.delete("/api/config/tool-keys/{name}", response_model=AIKeyDeleteResponse) async def api_delete_tool_key(name: str, current_user=Depends(require_admin)): """Delete a stored tool key (the environment fallback still applies).""" key_name = name.upper() try: existed = _delete_tool_key(key_name) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) logger.info(f"Tool key deleted: {key_name} (existed={existed})") return {"status": "deleted", "key": key_name} @app.post("/api/config/ai-keys/test", response_model=AITestResponse) async def api_test_ai_keys(current_user=Depends(require_admin)): """Test which AI providers are configured. Each provider has a dedicated (URL, header-name) test pair. - Most OpenAI-compatible APIs use `Authorization: Bearer KEY` - Xiaomi MiMo uses `api-key: KEY` - Gemini uses a query-string key """ results = {} for key_name, label, test_url_tmpl, header_name in [ # OpenAI-compatible — Authorization: Bearer ("DEEPSEEK_API_KEY", "deepseek", "https://api.deepseek.com/v1/models", "Authorization"), ("OPENROUTER_API_KEY","openrouter", "https://openrouter.ai/api/v1/models", "Authorization"), ("NVIDIA_API_KEY", "nvidia", "https://integrate.api.nvidia.com/v1/models", "Authorization"), ("QWENCLOUD_API_KEY", "qwencloud", "https://dashscope.aliyuncs.com/compatible-mode/v1/models", "Authorization"), ("MISTRAL_API_KEY", "mistral", "https://api.mistral.ai/v1/models", "Authorization"), # Xiaomi MiMo — dedicated api-key header (NOT Authorization: Bearer) ("XIAOMI_API_KEY", "xiaomi", "https://api.xiaomimimo.com/v1/models", "api-key"), # Gemini — key in query string ("GEMINI_API_KEY", "gemini", "https://generativelanguage.googleapis.com/v1beta/models?key={key}", None), ]: key = get_ai_key(key_name) if not key: results[label] = "non configuré" continue try: import urllib.request url = test_url_tmpl.replace("{key}", key) if "{key}" in test_url_tmpl else test_url_tmpl if header_name: req = urllib.request.Request(url, headers={header_name: key}) else: req = urllib.request.Request(url) urllib.request.urlopen(req, timeout=5) results[label] = "ok" except Exception as e: # Truncate the error to keep the response small. results[label] = "erreur: " + str(e)[:80] return results # --------------------------------------------------------------------------- # AI Models — list available models per provider # --------------------------------------------------------------------------- @app.get("/api/config/ai-models", response_model=AIModelsResponse) async def api_list_ai_models(provider: str = Query(...), current_user=Depends(require_admin)): """List available models for a given AI provider. Strategy: 1. Try the provider's public models endpoint (OpenAI-compatible /v1/models or Gemini). 2. If the network call fails (timeout, 4xx, 5xx, DNS, etc.), fall back to a curated static list of known-good models for that provider. 3. Always return a non-empty list when the provider is known, so the UI dropdown is never empty. """ provider = provider.lower() from backend.model_capabilities import get_capabilities_for_models from backend.provider_capabilities import remember_declared_capabilities all_providers = ("deepseek", "openrouter", "gemini", "nvidia", "qwencloud", "xiaomi", "mistral") if provider not in all_providers: return {"models": [], "error": f"Unknown provider: {provider}", "source": "validation"} key_name = f"{provider.upper()}_API_KEY" key = get_ai_key(key_name) if not key: # No key configured — return curated fallback list so the UI can # still show what WOULD be available once a key is set. fallback = _FALLBACK_MODELS.get(provider, []) return {"models": fallback, "source": "fallback", "capabilities": get_capabilities_for_models(provider, fallback), "note": "API key not configured — showing default model list"} # Build URL if provider == "gemini": url = f"https://generativelanguage.googleapis.com/v1beta/models?key={key}" elif provider == "deepseek": url = "https://api.deepseek.com/v1/models" elif provider == "openrouter": url = "https://openrouter.ai/api/v1/models" elif provider == "nvidia": url = "https://integrate.api.nvidia.com/v1/models" elif provider == "qwencloud": url = "https://dashscope.aliyuncs.com/compatible-mode/v1/models" elif provider == "xiaomi": # Xiaomi MiMo — dedicated api-key header (NOT Authorization: Bearer). # Endpoint: https://api.xiaomimimo.com/v1/models url = "https://api.xiaomimimo.com/v1/models" models = [] # parsed below with the custom header elif provider == "mistral": url = "https://api.mistral.ai/v1/models" try: if provider == "gemini": req = urllib.request.Request(url) elif provider == "xiaomi": # Xiaomi MiMo uses a dedicated api-key header. req = urllib.request.Request(url, headers={"api-key": key}) else: req = urllib.request.Request(url, headers={"Authorization": "Bearer " + key}) with urllib.request.urlopen(req, timeout=10) as resp: data = _json.loads(resp.read().decode()) if provider == "gemini": models = [m.get("name", "") for m in data.get("models", []) if m.get("name")] # Gemini returns names like "models/gemini-1.5-flash" — strip prefix models = [m.replace("models/", "") for m in models] else: models = [m.get("id", "") for m in data.get("data", []) if m.get("id")] # Cache the capabilities the provider declares for these models # (BUG-044) — get_capabilities_for_models() below then returns the # provider's own truth for the flags it declares, the curated table # for the rest. Providers that declare nothing are left untouched. remember_declared_capabilities(provider, data) if models: # Prepend the configured default if not already present default = PROVIDERS.get(provider, {}).get("model") if default and default not in models: models = [default] + models return {"models": models, "source": "live", "count": len(models), "capabilities": get_capabilities_for_models(provider, models)} # Empty list from API — fall through to fallback raise ValueError("empty model list from provider API") except Exception as e: # Network error, auth error, parsing error — use curated fallback fallback = _FALLBACK_MODELS.get(provider, []) return {"models": fallback, "source": "fallback", "error": str(e)[:200], "capabilities": get_capabilities_for_models(provider, fallback), "note": "Could not reach provider API — showing default model list"} # ── Curated fallback model lists ────────────────────────────────────────── # Used when the provider API is unreachable or returns empty. # Keep these short and focused on models known to work with the # OpenAI-compatible chat completions interface (or Gemini's generateContent). _FALLBACK_MODELS: dict[str, list[str]] = { "deepseek": [ "deepseek-chat", "deepseek-reasoner", ], "openrouter": [ "openai/gpt-4o-mini", "openai/gpt-4o", "anthropic/claude-3.5-sonnet", "anthropic/claude-3-haiku", "google/gemini-2.0-flash-exp:free", "meta-llama/llama-3.1-70b-instruct", "meta-llama/llama-3.1-8b-instruct:free", "mistralai/mistral-large-latest", ], "gemini": [ "gemini-2.0-flash", "gemini-2.0-flash-exp", "gemini-1.5-pro", "gemini-1.5-flash", "gemini-1.5-flash-8b", ], "nvidia": [ "meta/llama-3.1-405b-instruct", "meta/llama-3.1-70b-instruct", "meta/llama-3.1-8b-instruct", "mistralai/mistral-large", "google/gemma-2-27b-it", "nvidia/llama-3.1-nemotron-70b-instruct", ], "qwencloud": [ "qwen-max", "qwen-plus", "qwen-turbo", "qwen-long", "qwen-vl-max", "qwen-vl-plus", ], "xiaomi": [ # Xiaomi MiMo models — the public /v1/models endpoint requires the # `api-key` custom header (NOT Authorization: Bearer), so the live # call often fails with 401 even with the right key. We ship a # known-good list as fallback. See https://mimo.mi.com/docs/ "mimo-v2.5-pro", "mimo-v2.5", "mimo-v2.5-asr", "mimo-v2.5-tts", "mimo-v2.5-tts-voiceclone", "mimo-v2.5-tts-voicedesign", ], "mistral": [ "mistral-large-latest", "mistral-medium-latest", "mistral-small-latest", "open-mistral-7b", "open-mixtral-8x7b", "codestral-latest", ], } # --------------------------------------------------------------------------- # Diagnostics API # --------------------------------------------------------------------------- @app.get("/api/diagnostics", response_model=DiagnosticsResponse) async def api_diagnostics(current_user=Depends(require_admin)): """Return index statistics and system diagnostics. Includes document counts, token counts, memory estimates, and inverted index status. """ import sys from backend.search import get_inverted_index inv = get_inverted_index() # Per-vault stats vault_stats = {} total_files = 0 total_tags = 0 # Snapshot both dicts first: the indexer mutates them from background # threads, and iterating a live dict raises "dictionary changed size". for vname, vdata in list(index.items()): file_count = len(vdata.get("files", [])) tag_count = len(vdata.get("tags", {})) vault_stats[vname] = {"file_count": file_count, "tag_count": tag_count} total_files += file_count total_tags += tag_count # Memory estimate for inverted index word_index = inv.word_index.copy() word_index_entries = sum(len(docs) for docs in word_index.values()) mem_estimate_mb = round( (sys.getsizeof(inv.word_index) + word_index_entries * 80 + len(inv.doc_info) * 200 + len(inv._sorted_tokens) * 60) / (1024 * 1024), 2 ) return { "index": { "total_files": total_files, "total_tags": total_tags, "vaults": vault_stats, }, "inverted_index": { "unique_tokens": len(word_index), "total_postings": word_index_entries, "documents": inv.doc_count, "sorted_tokens": len(inv._sorted_tokens), "is_stale": inv.is_stale(), "memory_estimate_mb": mem_estimate_mb, }, "config": _load_config(), "search_executor": { "active": get_search_executor() is not None, "max_workers": get_search_executor()._max_workers if get_search_executor() else 0, }, } # --------------------------------------------------------------------------- # Dashboard endpoint (aggregated stats) # --------------------------------------------------------------------------- @app.get("/api/dashboard", response_model=DashboardResponse) async def api_dashboard(current_user=Depends(require_auth)): """Aggregated dashboard statistics across all accessible vaults.""" user_vaults = current_user.get("_token_vaults") or current_user.get("vaults", []) vault_stats = [] total_files = 0 total_tags = set() total_size = 0 total_images = 0 for vname, vdata in index.items(): if "*" not in user_vaults and vname not in user_vaults: continue files = vdata.get("files", []) fc = len(files) total_files += fc vtags = set() vsize = 0 vimages = 0 for f in files: vtags.update(f.get("tags", [])) vsize += f.get("size", 0) if (f.get("extension") or "").lower() in IMAGE_EXTENSIONS: vimages += 1 total_tags.update(vtags) total_size += vsize total_images += vimages vault_stats.append({ "name": vname, "file_count": fc, "tag_count": len(vtags), "total_size_bytes": vsize, "image_count": vimages, }) return { "vaults": vault_stats, "total_files": total_files, "total_tags": len(total_tags), "total_size_bytes": total_size, "total_images": total_images, } # --------------------------------------------------------------------------- # Webhook CRUD endpoints : voir backend.routers.webhooks (#85 T2) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Share (public document) endpoints : voir backend.routers.sharing (#85 T3) # --------------------------------------------------------------------------- # --------------------------------------------------------------------------- # Syncthing conflict endpoints # --------------------------------------------------------------------------- @app.get("/api/conflicts", response_model=ConflictsResponse) async def api_conflicts(current_user=Depends(require_auth)): """List sync-conflict files across accessible vaults.""" user_vaults = current_user.get("_token_vaults") or current_user.get("vaults", []) all_conflicts = get_conflicts() if "*" not in user_vaults: all_conflicts = [c for c in all_conflicts if c["vault"] in user_vaults] return {"conflicts": all_conflicts, "total": len(all_conflicts)} @app.post("/api/conflicts/resolve", response_model=ConflictResolveResponse) async def api_conflict_resolve(body: dict = Body(...), current_user=Depends(require_auth)): """Resolve a conflict: keep_local (delete conflict file) or keep_conflict (replace original).""" vault_name = body.get("vault") conflict_path = body.get("conflict_path") original_path = body.get("original_path") action = body.get("action") # "keep_local" or "keep_conflict" # mypy: narrow down from dict values assert isinstance(vault_name, str), "'vault' is required and must be a string" assert isinstance(conflict_path, str), "'conflict_path' is required and must be a string" assert isinstance(original_path, str), "'original_path' is required and must be a string" if not check_vault_access(vault_name, current_user): raise HTTPException(403, f"Accès refusé à la vault '{vault_name}'") vault_data = get_vault_data(vault_name) if not vault_data: raise HTTPException(404, "Vault not found") vault_root = Path(vault_data["path"]) conf_file = _resolve_safe_path(vault_root, conflict_path) orig_file = _resolve_safe_path(vault_root, original_path) if not conf_file.exists(): raise HTTPException(404, "Conflict file not found") try: if action == "keep_conflict": _backup_file(orig_file, vault_name, original_path) shutil.copy2(conf_file, orig_file) logger.info(f"Conflict resolved (keep_conflict): {conflict_path} → {original_path}") conf_file.unlink() await remove_single_file(vault_name, conflict_path) log_file_delete(current_user["username"], vault_name, conflict_path) await sse_manager.broadcast("file_deleted", {"vault": vault_name, "path": conflict_path}) return {"status": "resolved", "action": action} except Exception as e: raise HTTPException(500, f"Error resolving conflict: {e!s}") # --------------------------------------------------------------------------- # Real-time collaboration — WebSocket endpoint (ROADMAP #62) # --------------------------------------------------------------------------- @app.websocket("/ws/collab/{vault_name}/{path:path}") async def collab_websocket(websocket: WebSocket, vault_name: str, path: str): """Real-time collaborative editing over WebSocket (ROADMAP #62). One *room* is created per ``vault::path``; all clients editing the same file share Yjs/CRDT updates, awareness (cursors/selection) and a debounced server-side persistence of the markdown content. Authentication is performed manually (FastAPI ``Depends`` do not run for WebSocket routes) and vault access is enforced per connection. """ from backend.services.errors import ServiceError from backend.services.vaults import get_vault_root user = authenticate_websocket(websocket) if user is None: await websocket.close(code=4401) return if not check_vault_access(vault_name, user): await websocket.close(code=4403) return try: vault_root = get_vault_root(vault_name) file_path = _resolve_safe_path(vault_root, path) except ServiceError: await websocket.close(code=4404) return if not file_path.exists() or not file_path.is_file(): await websocket.close(code=4404) return await websocket.accept() await collab_manager.connect(websocket, vault_name, path, file_path, user) # --------------------------------------------------------------------------- # Static files & SPA fallback # --------------------------------------------------------------------------- if FRONTEND_DIR.exists(): # ``Cache-Control`` for /static is set by SecurityHeadersMiddleware (no-cache). app.mount("/static", StaticFiles(directory=str(FRONTEND_DIR)), name="static") @app.get("/sw.js") async def serve_service_worker(): """Serve the service worker for PWA support.""" sw_file = FRONTEND_DIR / "sw.js" if sw_file.exists(): return FileResponse( sw_file, media_type="application/javascript", headers={ "Cache-Control": "no-cache, no-store, must-revalidate", "Service-Worker-Allowed": "/" } ) raise HTTPException(status_code=404, detail="Service worker not found") @app.get("/manifest.json") async def serve_manifest(): """Serve the PWA manifest.""" manifest_file = FRONTEND_DIR / "manifest.json" if manifest_file.exists(): return FileResponse( manifest_file, media_type="application/manifest+json", headers={"Cache-Control": "no-cache"} ) raise HTTPException(status_code=404, detail="Manifest not found") @app.get("/popout/{vault_name}/{path:path}") async def serve_popout(vault_name: str, path: str): """Serve the minimalist popout page for a specific file.""" popout_file = FRONTEND_DIR / "popout.html" if popout_file.exists(): return HTMLResponse(content=popout_file.read_text(encoding="utf-8"), headers={"Cache-Control": "no-cache"}) raise HTTPException(status_code=404, detail="Popout template not found") @app.get("/editor-poc") async def serve_editor_poc(): """Serve the standalone Editor POC page (multi-zone toolbar demo).""" poc_file = FRONTEND_DIR / "editor-poc.html" if poc_file.exists(): return HTMLResponse(content=poc_file.read_text(encoding="utf-8"), headers={"Cache-Control": "no-cache"}) raise HTTPException(status_code=404, detail="Editor POC not found") @app.get("/admin.html", response_class=HTMLResponse) async def serve_admin_page(_current_user=Depends(require_admin)): """Serve the admin dashboard page (ROADMAP #71) — admin-gated. Must be declared BEFORE the SPA catch-all ``/{full_path:path}`` or the admin page would be shadowed by ``index.html`` (the reported bug: the Admin menu kept returning to the main page). """ admin_file = FRONTEND_DIR / "admin.html" if admin_file.exists(): return HTMLResponse(content=admin_file.read_text(encoding="utf-8"), headers={"Cache-Control": "no-cache"}) raise HTTPException(status_code=404, detail="Admin page not found") @app.get("/{full_path:path}") async def serve_spa(full_path: str): """Serve the SPA index.html for all non-API routes.""" index_file = FRONTEND_DIR / "index.html" if index_file.exists(): return HTMLResponse(content=index_file.read_text(encoding="utf-8"), headers={"Cache-Control": "no-cache"}) raise HTTPException(status_code=404, detail="Frontend not found")