ObsiGate/backend/routers/config.py

"""Configuration, AI keys, diagnostics & dashboard endpoints (ROADMAP #85, tranche 7).

Handlers déplacés depuis :mod:`backend.main` sans changement de
comportement : mêmes chemins (``/api/config*``, ``/api/diagnostics``,
``/api/dashboard``), mêmes modèles de réponse, mêmes dépendances
d'authentification.

Adaptations strictement équivalentes :
- ``_load_config`` / ``_save_config`` / ``_DEFAULT_CONFIG`` /
  ``_CONFIG_PATH`` / ``_BASE_DIR`` ont déménagé ici : ``main`` les
  réimporte pour son lifespan (pas de cycle : ce module ne dépend pas de
  ``main``).
- ``AI_KEYS_FILE`` / ``_write_ai_keys`` / ``_FALLBACK_MODELS`` ont déménagé
  ici (``AI_KEYS_FILE`` garde son chemin relatif ``data/api_keys.json``,
  résolu depuis le même CWD au runtime).
"""

import json as _json
import logging
import os
import urllib.request
from pathlib import Path

from fastapi import APIRouter, Body, Depends, HTTPException, Query

from backend.ai import PROVIDERS, _read_ai_keys, get_ai_key
from backend.auth.middleware import require_admin, require_auth
from backend.indexer import index
from backend.media_types import IMAGE_EXTENSIONS
from backend.schemas import (
    AIKeyDeleteResponse,
    AIKeysResponse,
    AIModelsResponse,
    AITestResponse,
    AppConfigResponse,
    DashboardResponse,
    DiagnosticsResponse,
    StatusResponse,
)
from backend.search_executor import get_search_executor
from backend.tools.secrets import (
    TOOL_KEY_NAMES as _TOOL_KEY_NAMES,
)
from backend.tools.secrets import (
    delete_tool_key as _delete_tool_key,
)
from backend.tools.secrets import (
    get_tool_key as _get_tool_key,
)
from backend.tools.secrets import (
    mask_value as _mask_tool_value,
)
from backend.tools.secrets import (
    set_tool_key as _set_tool_key,
)

logger = logging.getLogger("obsigate")

router = APIRouter(tags=["System"])

_BASE_DIR = Path(__file__).resolve().parent.parent.parent
_CONFIG_PATH = _BASE_DIR / "data" / "config.json"

_DEFAULT_CONFIG = {
    "search_workers": 2,
    "debounce_ms": 300,
    "results_per_page": 50,
    "min_query_length": 2,
    "search_timeout_ms": 30000,
    "max_content_size": 100000,
    "snippet_context_chars": 120,
    "max_snippet_highlights": 5,
    "title_boost": 3.0,
    "path_boost": 1.5,
    "watcher_enabled": True,
    "watcher_use_polling": False,
    "watcher_polling_interval": 5.0,
    "watcher_debounce": 2.0,
    "tag_boost": 2.0,
    "prefix_max_expansions": 50,
    "recent_files_limit": 20,
    "max_backups_per_file": 10,
    "ai_default_provider": "deepseek",
    "ai_default_models": {},
}


def _load_config() -> dict:
    """Load config from disk, merging with defaults."""
    config = dict(_DEFAULT_CONFIG)
    if _CONFIG_PATH.exists():
        try:
            stored = _json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
            config.update(stored)
        except Exception as e:
            logger.warning(f"Failed to read config.json: {e}")
    return config


def _save_config(config: dict) -> None:
    """Persist config to disk."""
    try:
        _CONFIG_PATH.write_text(
            _json.dumps(config, indent=2, ensure_ascii=False),
            encoding="utf-8",
        )
    except Exception as e:
        logger.error(f"Failed to write config.json: {e}")
        raise HTTPException(status_code=500, detail=f"Failed to save config: {e}")


AI_KEYS_FILE = Path("data/api_keys.json")

def _write_ai_keys(data: dict):
    AI_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True)
    tmp = AI_KEYS_FILE.with_suffix(".tmp")
    tmp.write_text(_json.dumps(data, indent=2), encoding="utf-8")
    tmp.replace(AI_KEYS_FILE)

@router.get("/api/config", response_model=AppConfigResponse)
async def api_get_config(current_user=Depends(require_auth)):
    """Return current configuration with defaults for missing keys."""
    return _load_config()


@router.post("/api/config", response_model=AppConfigResponse)
async def api_set_config(body: dict = Body(...), current_user=Depends(require_admin)):
    """Update configuration. Only known keys are accepted.

    Keys matching ``_DEFAULT_CONFIG`` are validated and persisted.
    Unknown keys are silently ignored.
    Returns the full merged config after update.
    """
    current = _load_config()
    updated_keys = []
    for key, value in body.items():
        if key in _DEFAULT_CONFIG:
            expected_type = type(_DEFAULT_CONFIG[key])
            if isinstance(value, expected_type) or (expected_type is float and isinstance(value, (int, float))):
                current[key] = value
                updated_keys.append(key)
            else:
                raise HTTPException(
                    status_code=400,
                    detail=f"Invalid type for '{key}': expected {expected_type.__name__}, got {type(value).__name__}",
                )
    _save_config(current)
    if any(k.startswith("ai_") for k in updated_keys):
        try:
            from backend.ai import reload_ai_config
            reload_ai_config()
        except Exception as e:
            logger.warning(f"Failed to reload AI config: {e}")
    logger.info(f"Config updated: {updated_keys}")
    return current


@router.get("/api/config/ai-keys", response_model=AIKeysResponse)
async def api_get_ai_keys(current_user=Depends(require_admin)):
    """Return stored AI keys (values masked)."""
    keys = _read_ai_keys()
    masked = {}
    for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]:
        val = keys.get(k, "") or os.environ.get(k, "")
        if val:
            masked[k] = val[:4] + "..." + val[-4:] if len(val) > 8 else "***"
        else:
            masked[k] = ""
    return masked

@router.post("/api/config/ai-keys", response_model=StatusResponse)
async def api_set_ai_keys(body: dict = Body(...), current_user=Depends(require_admin)):
    """Save AI keys. Pass {"DEEPSEEK_API_KEY":"sk-...","OPENROUTER_API_KEY":"...","GEMINI_API_KEY":"..."}"""
    keys = _read_ai_keys()
    for k in ["DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"]:
        if body.get(k):
            keys[k] = body[k]
    _write_ai_keys(keys)
    logger.info("AI keys updated")
    return {"status": "ok"}


@router.delete("/api/config/ai-keys/{provider_env}", response_model=AIKeyDeleteResponse)
async def api_delete_ai_key(provider_env: str, current_user=Depends(require_admin)):
    """Delete a specific AI provider key from storage."""
    allowed = {"DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY",
               "NVIDIA_API_KEY", "QWENCLOUD_API_KEY", "XIAOMI_API_KEY", "MISTRAL_API_KEY"}
    key_name = provider_env.upper()
    if key_name not in allowed:
        raise HTTPException(status_code=400, detail=f"Clé inconnue: {provider_env}")
    keys = _read_ai_keys()
    if key_name in keys:
        del keys[key_name]
        _write_ai_keys(keys)
    # Also clear from env at runtime so get_ai_key() no longer finds it
    os.environ.pop(key_name, None)
    logger.info(f"AI key deleted: {key_name}")
    return {"status": "deleted", "key": key_name}


@router.get("/api/config/tool-keys", response_model=AIKeysResponse)
async def api_get_tool_keys(current_user=Depends(require_admin)):
    """Return tool/connected-source configuration (tokens masked, URLs clear)."""
    masked = {}
    for name in _TOOL_KEY_NAMES:
        masked[name] = _mask_tool_value(name, _get_tool_key(name))
    return masked


@router.post("/api/config/tool-keys", response_model=StatusResponse)
async def api_set_tool_keys(body: dict = Body(...), current_user=Depends(require_admin)):
    """Save tool/connected-source keys.

    Only whitelisted names (``backend.tools.secrets.TOOL_KEY_NAMES``) are
    accepted: Tavily/Brave/SerpAPI/Exa API keys, Gitea URL + token, GitHub
    token. Empty values delete the stored entry.
    """
    updated = []
    for name, value in body.items():
        if name not in _TOOL_KEY_NAMES:
            raise HTTPException(status_code=400, detail=f"Clé inconnue: {name}")
        if value is not None and not isinstance(value, str):
            raise HTTPException(status_code=400, detail=f"Type invalide pour {name}")
        _set_tool_key(name, value or "")
        updated.append(name)
    logger.info(f"Tool keys updated: {updated}")
    return {"status": "ok"}


@router.delete("/api/config/tool-keys/{name}", response_model=AIKeyDeleteResponse)
async def api_delete_tool_key(name: str, current_user=Depends(require_admin)):
    """Delete a stored tool key (the environment fallback still applies)."""
    key_name = name.upper()
    try:
        existed = _delete_tool_key(key_name)
    except ValueError as e:
        raise HTTPException(status_code=400, detail=str(e))
    logger.info(f"Tool key deleted: {key_name} (existed={existed})")
    return {"status": "deleted", "key": key_name}


@router.post("/api/config/ai-keys/test", response_model=AITestResponse)
async def api_test_ai_keys(current_user=Depends(require_admin)):
    """Test which AI providers are configured.

    Each provider has a dedicated (URL, header-name) test pair.
    - Most OpenAI-compatible APIs use `Authorization: Bearer KEY`
    - Xiaomi MiMo uses `api-key: KEY`
    - Gemini uses a query-string key
    """
    results = {}
    for key_name, label, test_url_tmpl, header_name in [
        # OpenAI-compatible — Authorization: Bearer
        ("DEEPSEEK_API_KEY",  "deepseek",   "https://api.deepseek.com/v1/models",                   "Authorization"),
        ("OPENROUTER_API_KEY","openrouter", "https://openrouter.ai/api/v1/models",                   "Authorization"),
        ("NVIDIA_API_KEY",    "nvidia",     "https://integrate.api.nvidia.com/v1/models",           "Authorization"),
        ("QWENCLOUD_API_KEY", "qwencloud",  "https://dashscope.aliyuncs.com/compatible-mode/v1/models", "Authorization"),
        ("MISTRAL_API_KEY",   "mistral",    "https://api.mistral.ai/v1/models",                     "Authorization"),
        # Xiaomi MiMo — dedicated api-key header (NOT Authorization: Bearer)
        ("XIAOMI_API_KEY",    "xiaomi",     "https://api.xiaomimimo.com/v1/models",                 "api-key"),
        # Gemini — key in query string
        ("GEMINI_API_KEY",    "gemini",     "https://generativelanguage.googleapis.com/v1beta/models?key={key}", None),
    ]:
        key = get_ai_key(key_name)
        if not key:
            results[label] = "non configuré"
            continue
        try:
            url = test_url_tmpl.replace("{key}", key) if "{key}" in test_url_tmpl else test_url_tmpl
            if header_name:
                req = urllib.request.Request(url, headers={header_name: key})
            else:
                req = urllib.request.Request(url)
            urllib.request.urlopen(req, timeout=5)
            results[label] = "ok"
        except Exception as e:
            # Truncate the error to keep the response small.
            results[label] = "erreur: " + str(e)[:80]
    return results


@router.get("/api/config/ai-models", response_model=AIModelsResponse)
async def api_list_ai_models(provider: str = Query(...), current_user=Depends(require_admin)):
    """List available models for a given AI provider.

    Strategy:
    1. Try the provider's public models endpoint (OpenAI-compatible /v1/models or Gemini).
    2. If the network call fails (timeout, 4xx, 5xx, DNS, etc.), fall back to a
       curated static list of known-good models for that provider.
    3. Always return a non-empty list when the provider is known, so the UI
       dropdown is never empty.
    """
    provider = provider.lower()

    from backend.model_capabilities import get_capabilities_for_models
    from backend.provider_capabilities import remember_declared_capabilities

    all_providers = ("deepseek", "openrouter", "gemini", "nvidia", "qwencloud", "xiaomi", "mistral")
    if provider not in all_providers:
        return {"models": [], "error": f"Unknown provider: {provider}", "source": "validation"}

    key_name = f"{provider.upper()}_API_KEY"
    key = get_ai_key(key_name)
    if not key:
        # No key configured — return curated fallback list so the UI can
        # still show what WOULD be available once a key is set.
        fallback = _FALLBACK_MODELS.get(provider, [])
        return {"models": fallback, "source": "fallback",
                "capabilities": get_capabilities_for_models(provider, fallback),
                "note": "API key not configured — showing default model list"}

    # Build URL
    if provider == "gemini":
        url = f"https://generativelanguage.googleapis.com/v1beta/models?key={key}"
    elif provider == "deepseek":
        url = "https://api.deepseek.com/v1/models"
    elif provider == "openrouter":
        url = "https://openrouter.ai/api/v1/models"
    elif provider == "nvidia":
        url = "https://integrate.api.nvidia.com/v1/models"
    elif provider == "qwencloud":
        url = "https://dashscope.aliyuncs.com/compatible-mode/v1/models"
    elif provider == "xiaomi":
        # Xiaomi MiMo — dedicated api-key header (NOT Authorization: Bearer).
        # Endpoint: https://api.xiaomimimo.com/v1/models
        url = "https://api.xiaomimimo.com/v1/models"
        models = []  # parsed below with the custom header
    elif provider == "mistral":
        url = "https://api.mistral.ai/v1/models"

    try:
        if provider == "gemini":
            req = urllib.request.Request(url)
        elif provider == "xiaomi":
            # Xiaomi MiMo uses a dedicated api-key header.
            req = urllib.request.Request(url, headers={"api-key": key})
        else:
            req = urllib.request.Request(url, headers={"Authorization": "Bearer " + key})

        with urllib.request.urlopen(req, timeout=10) as resp:
            data = _json.loads(resp.read().decode())

        if provider == "gemini":
            models = [m.get("name", "") for m in data.get("models", []) if m.get("name")]
            # Gemini returns names like "models/gemini-1.5-flash" — strip prefix
            models = [m.replace("models/", "") for m in models]
        else:
            models = [m.get("id", "") for m in data.get("data", []) if m.get("id")]

        # Cache the capabilities the provider declares for these models
        # (BUG-044) — get_capabilities_for_models() below then returns the
        # provider's own truth for the flags it declares, the curated table
        # for the rest. Providers that declare nothing are left untouched.
        remember_declared_capabilities(provider, data)

        if models:
            # Prepend the configured default if not already present
            default = PROVIDERS.get(provider, {}).get("model")
            if default and default not in models:
                models = [default] + models
            return {"models": models, "source": "live", "count": len(models),
                    "capabilities": get_capabilities_for_models(provider, models)}
        # Empty list from API — fall through to fallback
        raise ValueError("empty model list from provider API")
    except Exception as e:
        # Network error, auth error, parsing error — use curated fallback
        fallback = _FALLBACK_MODELS.get(provider, [])
        return {"models": fallback, "source": "fallback", "error": str(e)[:200],
                "capabilities": get_capabilities_for_models(provider, fallback),
                "note": "Could not reach provider API — showing default model list"}


# ── Curated fallback model lists ──────────────────────────────────────────
# Used when the provider API is unreachable or returns empty.
# Keep these short and focused on models known to work with the
# OpenAI-compatible chat completions interface (or Gemini's generateContent).
_FALLBACK_MODELS: dict[str, list[str]] = {
    "deepseek": [
        "deepseek-chat",
        "deepseek-reasoner",
    ],
    "openrouter": [
        "openai/gpt-4o-mini",
        "openai/gpt-4o",
        "anthropic/claude-3.5-sonnet",
        "anthropic/claude-3-haiku",
        "google/gemini-2.0-flash-exp:free",
        "meta-llama/llama-3.1-70b-instruct",
        "meta-llama/llama-3.1-8b-instruct:free",
        "mistralai/mistral-large-latest",
    ],
    "gemini": [
        "gemini-2.0-flash",
        "gemini-2.0-flash-exp",
        "gemini-1.5-pro",
        "gemini-1.5-flash",
        "gemini-1.5-flash-8b",
    ],
    "nvidia": [
        "meta/llama-3.1-405b-instruct",
        "meta/llama-3.1-70b-instruct",
        "meta/llama-3.1-8b-instruct",
        "mistralai/mistral-large",
        "google/gemma-2-27b-it",
        "nvidia/llama-3.1-nemotron-70b-instruct",
    ],
    "qwencloud": [
        "qwen-max",
        "qwen-plus",
        "qwen-turbo",
        "qwen-long",
        "qwen-vl-max",
        "qwen-vl-plus",
    ],
    "xiaomi": [
        # Xiaomi MiMo models — the public /v1/models endpoint requires the
        # `api-key` custom header (NOT Authorization: Bearer), so the live
        # call often fails with 401 even with the right key. We ship a
        # known-good list as fallback. See https://mimo.mi.com/docs/
        "mimo-v2.5-pro",
        "mimo-v2.5",
        "mimo-v2.5-asr",
        "mimo-v2.5-tts",
        "mimo-v2.5-tts-voiceclone",
        "mimo-v2.5-tts-voicedesign",
    ],
    "mistral": [
        "mistral-large-latest",
        "mistral-medium-latest",
        "mistral-small-latest",
        "open-mistral-7b",
        "open-mixtral-8x7b",
        "codestral-latest",
    ],
}


@router.get("/api/diagnostics", response_model=DiagnosticsResponse)
async def api_diagnostics(current_user=Depends(require_admin)):
    """Return index statistics and system diagnostics.

    Includes document counts, token counts, memory estimates,
    and inverted index status.
    """
    import sys

    from backend.search import get_inverted_index

    inv = get_inverted_index()

    # Per-vault stats
    vault_stats = {}
    total_files = 0
    total_tags = 0
    # Snapshot both dicts first: the indexer mutates them from background
    # threads, and iterating a live dict raises "dictionary changed size".
    for vname, vdata in list(index.items()):
        file_count = len(vdata.get("files", []))
        tag_count = len(vdata.get("tags", {}))
        vault_stats[vname] = {"file_count": file_count, "tag_count": tag_count}
        total_files += file_count
        total_tags += tag_count

    # Memory estimate for inverted index
    word_index = inv.word_index.copy()
    word_index_entries = sum(len(docs) for docs in word_index.values())
    mem_estimate_mb = round(
        (sys.getsizeof(inv.word_index) + word_index_entries * 80
         + len(inv.doc_info) * 200
         + len(inv._sorted_tokens) * 60) / (1024 * 1024), 2
    )

    return {
        "index": {
            "total_files": total_files,
            "total_tags": total_tags,
            "vaults": vault_stats,
        },
        "inverted_index": {
            "unique_tokens": len(word_index),
            "total_postings": word_index_entries,
            "documents": inv.doc_count,
            "sorted_tokens": len(inv._sorted_tokens),
            "is_ready": inv.is_ready(),
            "memory_estimate_mb": mem_estimate_mb,
        },
        "config": _load_config(),
        "search_executor": {
            "active": get_search_executor() is not None,
            "max_workers": get_search_executor()._max_workers if get_search_executor() else 0,
        },
    }


@router.get("/api/dashboard", response_model=DashboardResponse)
async def api_dashboard(current_user=Depends(require_auth)):
    """Aggregated dashboard statistics across all accessible vaults."""
    user_vaults = current_user.get("_token_vaults") or current_user.get("vaults", [])
    vault_stats = []
    total_files = 0
    total_tags = set()
    total_size = 0
    total_images = 0
    for vname, vdata in index.items():
        if "*" not in user_vaults and vname not in user_vaults:
            continue
        files = vdata.get("files", [])
        fc = len(files)
        total_files += fc
        vtags = set()
        vsize = 0
        vimages = 0
        for f in files:
            vtags.update(f.get("tags", []))
            vsize += f.get("size", 0)
            if (f.get("extension") or "").lower() in IMAGE_EXTENSIONS:
                vimages += 1
        total_tags.update(vtags)
        total_size += vsize
        total_images += vimages
        vault_stats.append({
            "name": vname, "file_count": fc, "tag_count": len(vtags),
            "total_size_bytes": vsize, "image_count": vimages,
        })
    return {
        "vaults": vault_stats,
        "total_files": total_files,
        "total_tags": len(total_tags),
        "total_size_bytes": total_size,
        "total_images": total_images,
    }