"""Configuration, AI keys, diagnostics & dashboard endpoints (ROADMAP #85, tranche 7). Handlers déplacés depuis :mod:`backend.main` sans changement de comportement : mêmes chemins (``/api/config*``, ``/api/diagnostics``, ``/api/dashboard``), mêmes modèles de réponse, mêmes dépendances d'authentification. Adaptations strictement équivalentes : - ``_load_config`` / ``_save_config`` / ``_DEFAULT_CONFIG`` / ``_CONFIG_PATH`` / ``_BASE_DIR`` ont déménagé ici : ``main`` les réimporte pour son lifespan (pas de cycle : ce module ne dépend pas de ``main``). - ``AI_KEYS_FILE`` / ``_write_ai_keys`` / ``_FALLBACK_MODELS`` ont déménagé ici (``AI_KEYS_FILE`` garde son chemin relatif ``data/api_keys.json``, résolu depuis le même CWD au runtime). """ import json as _json import logging import os import urllib.request from pathlib import Path from fastapi import APIRouter, Body, Depends, HTTPException, Query from backend.ai import PROVIDERS, _read_ai_keys, get_ai_key from backend.auth.middleware import check_vault_access, require_admin, require_auth from backend.indexer import index from backend.media_types import IMAGE_EXTENSIONS from backend.schemas import ( AIKeyDeleteResponse, AIKeysResponse, AIModelsResponse, AITestResponse, AppConfigResponse, DashboardResponse, DiagnosticsResponse, StatusResponse, ) from backend.search_executor import get_search_executor from backend.tools.secrets import ( TOOL_KEY_NAMES as _TOOL_KEY_NAMES, ) from backend.tools.secrets import ( delete_tool_key as _delete_tool_key, ) from backend.tools.secrets import ( get_tool_key as _get_tool_key, ) from backend.tools.secrets import ( mask_value as _mask_tool_value, ) from backend.tools.secrets import ( set_tool_key as _set_tool_key, ) logger = logging.getLogger("obsigate") router = APIRouter(tags=["System"]) _BASE_DIR = Path(__file__).resolve().parent.parent.parent _CONFIG_PATH = _BASE_DIR / "data" / "config.json" _DEFAULT_CONFIG = { "search_workers": 2, "debounce_ms": 300, "results_per_page": 50, "min_query_length": 2, "search_timeout_ms": 30000, "max_content_size": 100000, "snippet_context_chars": 120, "max_snippet_highlights": 5, "title_boost": 3.0, "path_boost": 1.5, "watcher_enabled": True, "watcher_use_polling": False, "watcher_polling_interval": 5.0, "watcher_debounce": 2.0, "tag_boost": 2.0, "prefix_max_expansions": 50, "recent_files_limit": 20, "max_backups_per_file": 10, "ai_default_provider": "deepseek", "ai_default_models": {}, } def _load_config() -> dict: """Load config from disk, merging with defaults.""" config = dict(_DEFAULT_CONFIG) if _CONFIG_PATH.exists(): try: stored = _json.loads(_CONFIG_PATH.read_text(encoding="utf-8")) config.update(stored) except Exception as e: logger.warning(f"Failed to read config.json: {e}") return config def _save_config(config: dict) -> None: """Persist config to disk.""" try: _CONFIG_PATH.write_text( _json.dumps(config, indent=2, ensure_ascii=False), encoding="utf-8", ) except Exception as e: logger.error(f"Failed to write config.json: {e}") raise HTTPException(status_code=500, detail=f"Failed to save config: {e}") AI_KEYS_FILE = Path("data/api_keys.json") def _write_ai_keys(data: dict): AI_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True) tmp = AI_KEYS_FILE.with_suffix(".tmp") tmp.write_text(_json.dumps(data, indent=2), encoding="utf-8") tmp.replace(AI_KEYS_FILE) @router.get("/api/config", response_model=AppConfigResponse) async def api_get_config(current_user=Depends(require_auth)): """Return current configuration with defaults for missing keys.""" return _load_config() @router.post("/api/config", response_model=AppConfigResponse) async def api_set_config(body: dict = Body(...), current_user=Depends(require_admin)): """Update configuration. Only known keys are accepted. Keys matching ``_DEFAULT_CONFIG`` are validated and persisted. Unknown keys are silently ignored. Returns the full merged config after update. """ current = _load_config() updated_keys = [] for key, value in body.items(): if key in _DEFAULT_CONFIG: expected_type = type(_DEFAULT_CONFIG[key]) if isinstance(value, expected_type) or (expected_type is float and isinstance(value, (int, float))): current[key] = value updated_keys.append(key) else: raise HTTPException( status_code=400, detail=f"Invalid type for '{key}': expected {expected_type.__name__}, got {type(value).__name__}", ) _save_config(current) if any(k.startswith("ai_") for k in updated_keys): try: from backend.ai import reload_ai_config reload_ai_config() except Exception as e: logger.warning(f"Failed to reload AI config: {e}") logger.info(f"Config updated: {updated_keys}") return current @router.get("/api/config/ai-keys", response_model=AIKeysResponse) async def api_get_ai_keys(current_user=Depends(require_admin)): """Return stored AI keys (values masked).""" keys = _read_ai_keys() masked = {} for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]: val = keys.get(k, "") or os.environ.get(k, "") if val: masked[k] = val[:4] + "..." + val[-4:] if len(val) > 8 else "***" else: masked[k] = "" return masked @router.post("/api/config/ai-keys", response_model=StatusResponse) async def api_set_ai_keys(body: dict = Body(...), current_user=Depends(require_admin)): """Save AI keys. Pass {"DEEPSEEK_API_KEY":"sk-...","OPENROUTER_API_KEY":"...","GEMINI_API_KEY":"..."}""" keys = _read_ai_keys() for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]: if body.get(k): keys[k] = body[k] _write_ai_keys(keys) logger.info("AI keys updated") return {"status": "ok"} @router.delete("/api/config/ai-keys/{provider_env}", response_model=AIKeyDeleteResponse) async def api_delete_ai_key(provider_env: str, current_user=Depends(require_admin)): """Delete a specific AI provider key from storage.""" allowed = {"DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"} key_name = provider_env.upper() if key_name not in allowed: raise HTTPException(status_code=400, detail=f"Clé inconnue: {provider_env}") keys = _read_ai_keys() if key_name in keys: del keys[key_name] _write_ai_keys(keys) # Also clear from env at runtime so get_ai_key() no longer finds it os.environ.pop(key_name, None) logger.info(f"AI key deleted: {key_name}") return {"status": "deleted", "key": key_name} @router.get("/api/config/tool-keys", response_model=AIKeysResponse) async def api_get_tool_keys(current_user=Depends(require_admin)): """Return tool/connected-source configuration (tokens masked, URLs clear).""" masked = {} for name in _TOOL_KEY_NAMES: masked[name] = _mask_tool_value(name, _get_tool_key(name)) return masked @router.post("/api/config/tool-keys", response_model=StatusResponse) async def api_set_tool_keys(body: dict = Body(...), current_user=Depends(require_admin)): """Save tool/connected-source keys. Only whitelisted names (``backend.tools.secrets.TOOL_KEY_NAMES``) are accepted: Tavily/Brave/SerpAPI/Exa API keys, Gitea URL + token, GitHub token. Empty values delete the stored entry. """ updated = [] for name, value in body.items(): if name not in _TOOL_KEY_NAMES: raise HTTPException(status_code=400, detail=f"Clé inconnue: {name}") if value is not None and not isinstance(value, str): raise HTTPException(status_code=400, detail=f"Type invalide pour {name}") _set_tool_key(name, value or "") updated.append(name) logger.info(f"Tool keys updated: {updated}") return {"status": "ok"} @router.delete("/api/config/tool-keys/{name}", response_model=AIKeyDeleteResponse) async def api_delete_tool_key(name: str, current_user=Depends(require_admin)): """Delete a stored tool key (the environment fallback still applies).""" key_name = name.upper() try: existed = _delete_tool_key(key_name) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) logger.info(f"Tool key deleted: {key_name} (existed={existed})") return {"status": "deleted", "key": key_name} @router.post("/api/config/ai-keys/test", response_model=AITestResponse) async def api_test_ai_keys(current_user=Depends(require_admin)): """Test which AI providers are configured. Each provider has a dedicated (URL, header-name) test pair. - Most OpenAI-compatible APIs use `Authorization: Bearer KEY` - Xiaomi MiMo uses `api-key: KEY` - Gemini uses a query-string key """ results = {} for key_name, label, test_url_tmpl, header_name in [ # OpenAI-compatible — Authorization: Bearer ("DEEPSEEK_API_KEY", "deepseek", "https://api.deepseek.com/v1/models", "Authorization"), ("OPENROUTER_API_KEY","openrouter", "https://openrouter.ai/api/v1/models", "Authorization"), ("NVIDIA_API_KEY", "nvidia", "https://integrate.api.nvidia.com/v1/models", "Authorization"), ("QWENCLOUD_API_KEY", "qwencloud", "https://dashscope.aliyuncs.com/compatible-mode/v1/models", "Authorization"), ("MISTRAL_API_KEY", "mistral", "https://api.mistral.ai/v1/models", "Authorization"), # Xiaomi MiMo — dedicated api-key header (NOT Authorization: Bearer) ("XIAOMI_API_KEY", "xiaomi", "https://api.xiaomimimo.com/v1/models", "api-key"), # Gemini — key in query string ("GEMINI_API_KEY", "gemini", "https://generativelanguage.googleapis.com/v1beta/models?key={key}", None), ]: key = get_ai_key(key_name) if not key: results[label] = "non configuré" continue try: url = test_url_tmpl.replace("{key}", key) if "{key}" in test_url_tmpl else test_url_tmpl if header_name: req = urllib.request.Request(url, headers={header_name: key}) else: req = urllib.request.Request(url) urllib.request.urlopen(req, timeout=5) results[label] = "ok" except Exception as e: # Truncate the error to keep the response small. results[label] = "erreur: " + str(e)[:80] return results @router.get("/api/config/ai-models", response_model=AIModelsResponse) async def api_list_ai_models(provider: str = Query(...), current_user=Depends(require_admin)): """List available models for a given AI provider. Strategy: 1. Try the provider's public models endpoint (OpenAI-compatible /v1/models or Gemini). 2. If the network call fails (timeout, 4xx, 5xx, DNS, etc.), fall back to a curated static list of known-good models for that provider. 3. Always return a non-empty list when the provider is known, so the UI dropdown is never empty. """ provider = provider.lower() from backend.model_capabilities import get_capabilities_for_models from backend.provider_capabilities import remember_declared_capabilities all_providers = ("deepseek", "openrouter", "gemini", "nvidia", "qwencloud", "xiaomi", "mistral") if provider not in all_providers: return {"models": [], "error": f"Unknown provider: {provider}", "source": "validation"} key_name = f"{provider.upper()}_API_KEY" key = get_ai_key(key_name) if not key: # No key configured — return curated fallback list so the UI can # still show what WOULD be available once a key is set. fallback = _FALLBACK_MODELS.get(provider, []) return {"models": fallback, "source": "fallback", "capabilities": get_capabilities_for_models(provider, fallback), "note": "API key not configured — showing default model list"} # Build URL if provider == "gemini": url = f"https://generativelanguage.googleapis.com/v1beta/models?key={key}" elif provider == "deepseek": url = "https://api.deepseek.com/v1/models" elif provider == "openrouter": url = "https://openrouter.ai/api/v1/models" elif provider == "nvidia": url = "https://integrate.api.nvidia.com/v1/models" elif provider == "qwencloud": url = "https://dashscope.aliyuncs.com/compatible-mode/v1/models" elif provider == "xiaomi": # Xiaomi MiMo — dedicated api-key header (NOT Authorization: Bearer). # Endpoint: https://api.xiaomimimo.com/v1/models url = "https://api.xiaomimimo.com/v1/models" models = [] # parsed below with the custom header elif provider == "mistral": url = "https://api.mistral.ai/v1/models" try: if provider == "gemini": req = urllib.request.Request(url) elif provider == "xiaomi": # Xiaomi MiMo uses a dedicated api-key header. req = urllib.request.Request(url, headers={"api-key": key}) else: req = urllib.request.Request(url, headers={"Authorization": "Bearer " + key}) with urllib.request.urlopen(req, timeout=10) as resp: data = _json.loads(resp.read().decode()) if provider == "gemini": models = [m.get("name", "") for m in data.get("models", []) if m.get("name")] # Gemini returns names like "models/gemini-1.5-flash" — strip prefix models = [m.replace("models/", "") for m in models] else: models = [m.get("id", "") for m in data.get("data", []) if m.get("id")] # Cache the capabilities the provider declares for these models # (BUG-044) — get_capabilities_for_models() below then returns the # provider's own truth for the flags it declares, the curated table # for the rest. Providers that declare nothing are left untouched. remember_declared_capabilities(provider, data) if models: # Prepend the configured default if not already present default = PROVIDERS.get(provider, {}).get("model") if default and default not in models: models = [default] + models return {"models": models, "source": "live", "count": len(models), "capabilities": get_capabilities_for_models(provider, models)} # Empty list from API — fall through to fallback raise ValueError("empty model list from provider API") except Exception as e: # Network error, auth error, parsing error — use curated fallback fallback = _FALLBACK_MODELS.get(provider, []) return {"models": fallback, "source": "fallback", "error": str(e)[:200], "capabilities": get_capabilities_for_models(provider, fallback), "note": "Could not reach provider API — showing default model list"} # ── Curated fallback model lists ────────────────────────────────────────── # Used when the provider API is unreachable or returns empty. # Keep these short and focused on models known to work with the # OpenAI-compatible chat completions interface (or Gemini's generateContent). _FALLBACK_MODELS: dict[str, list[str]] = { "deepseek": [ "deepseek-chat", "deepseek-reasoner", ], "openrouter": [ "openai/gpt-4o-mini", "openai/gpt-4o", "anthropic/claude-3.5-sonnet", "anthropic/claude-3-haiku", "google/gemini-2.0-flash-exp:free", "meta-llama/llama-3.1-70b-instruct", "meta-llama/llama-3.1-8b-instruct:free", "mistralai/mistral-large-latest", ], "gemini": [ "gemini-2.0-flash", "gemini-2.0-flash-exp", "gemini-1.5-pro", "gemini-1.5-flash", "gemini-1.5-flash-8b", ], "nvidia": [ "meta/llama-3.1-405b-instruct", "meta/llama-3.1-70b-instruct", "meta/llama-3.1-8b-instruct", "mistralai/mistral-large", "google/gemma-2-27b-it", "nvidia/llama-3.1-nemotron-70b-instruct", ], "qwencloud": [ "qwen-max", "qwen-plus", "qwen-turbo", "qwen-long", "qwen-vl-max", "qwen-vl-plus", ], "xiaomi": [ # Xiaomi MiMo models — the public /v1/models endpoint requires the # `api-key` custom header (NOT Authorization: Bearer), so the live # call often fails with 401 even with the right key. We ship a # known-good list as fallback. See https://mimo.mi.com/docs/ "mimo-v2.5-pro", "mimo-v2.5", "mimo-v2.5-asr", "mimo-v2.5-tts", "mimo-v2.5-tts-voiceclone", "mimo-v2.5-tts-voicedesign", ], "mistral": [ "mistral-large-latest", "mistral-medium-latest", "mistral-small-latest", "open-mistral-7b", "open-mixtral-8x7b", "codestral-latest", ], } @router.get("/api/diagnostics", response_model=DiagnosticsResponse) async def api_diagnostics(current_user=Depends(require_admin)): """Return index statistics and system diagnostics. Includes document counts, token counts, memory estimates, and inverted index status. """ import sys from backend.search import get_inverted_index inv = get_inverted_index() # Per-vault stats vault_stats = {} total_files = 0 total_tags = 0 # Snapshot both dicts first: the indexer mutates them from background # threads, and iterating a live dict raises "dictionary changed size". for vname, vdata in list(index.items()): file_count = len(vdata.get("files", [])) tag_count = len(vdata.get("tags", {})) vault_stats[vname] = {"file_count": file_count, "tag_count": tag_count} total_files += file_count total_tags += tag_count # Memory estimate for inverted index word_index = inv.word_index.copy() word_index_entries = sum(len(docs) for docs in word_index.values()) mem_estimate_mb = round( (sys.getsizeof(inv.word_index) + word_index_entries * 80 + len(inv.doc_info) * 200 + len(inv._sorted_tokens) * 60) / (1024 * 1024), 2 ) return { "index": { "total_files": total_files, "total_tags": total_tags, "vaults": vault_stats, }, "inverted_index": { "unique_tokens": len(word_index), "total_postings": word_index_entries, "documents": inv.doc_count, "sorted_tokens": len(inv._sorted_tokens), "is_ready": inv.is_ready(), "memory_estimate_mb": mem_estimate_mb, }, "config": _load_config(), "search_executor": { "active": get_search_executor() is not None, "max_workers": get_search_executor()._max_workers if get_search_executor() else 0, }, } @router.get("/api/dashboard", response_model=DashboardResponse) async def api_dashboard(current_user=Depends(require_auth)): """Aggregated dashboard statistics across all accessible vaults.""" vault_stats = [] total_files = 0 total_tags = set() total_size = 0 total_images = 0 for vname, vdata in index.items(): if not check_vault_access(vname, current_user): continue files = vdata.get("files", []) fc = len(files) total_files += fc vtags = set() vsize = 0 vimages = 0 for f in files: vtags.update(f.get("tags", [])) vsize += f.get("size", 0) if (f.get("extension") or "").lower() in IMAGE_EXTENSIONS: vimages += 1 total_tags.update(vtags) total_size += vsize total_images += vimages vault_stats.append({ "name": vname, "file_count": fc, "tag_count": len(vtags), "total_size_bytes": vsize, "image_count": vimages, }) return { "vaults": vault_stats, "total_files": total_files, "total_tags": len(total_tags), "total_size_bytes": total_size, "total_images": total_images, }