fix(ai): BUG-044 capacités des modèles lues chez le fournisseur (Mistral vision)
Le panneau de modèle par défaut et la bulle ⓘ n'affichaient aucun modèle Mistral « Vision capable » alors que GET api.mistral.ai/v1/models en déclare 28 : la table de capacités était entièrement statique et aucun de ses motifs ne correspondait aux familles Mistral actuelles (seul `pixtral`, retiré de l'API, les matchait). - backend/provider_capabilities.py (nouveau) : capacités déclarées par le fournisseur (Mistral `capabilities`, OpenRouter `architecture`), détectées par la forme du payload, snapshot en cache process-wide (TTL 30 min, surchargeable par AI_CAPABILITIES_TTL_SECONDS) rempli par GET /api/config/ai-models. - backend/model_capabilities.py : une déclaration prime sur la table statique pour chaque drapeau mentionné ; la table ne comble que le reste (Mistral ne déclare jamais `embedding`). Table corrigée pour le repli hors ligne : familles vision Mistral (ministral, magistral, mistral-small, mistral-medium, mistral-vibe-cli, labs-leanstral), mistral-ocr = vision sans chat, et défaut du fournisseur Mistral sans `embeddings` (mistral-large / codestral n'étaient plus des « embedders »). - backend/ai_routes.py : GET /api/ai/model-capabilities reste sans appel réseau (cache froid → table statique). - Tests : tests/test_provider_capabilities.py (nouveau), TestMistralFamilies et TestDeclaredCapabilities (bout en bout via l'API). - Docs : CHANGELOG [Unreleased], registre + journal ISSUES_TODOLIST, fiche docs/features/ai-provider-picker.md (§L). Vérifié : 28/28 modèles vision déclarés par Mistral détectés (0 avant), 0 écart dans les deux sens ; pytest 1007 passed / 6 skipped ; ruff 0 (backend) ; mypy 0 ; tests frontend unit 9/9 + IA 66/66 + validate-imports 37 modules ; instance de test reconstruite et vérifiée sur http://localhost:2020.
This commit is contained in:
@@ -14,6 +14,16 @@ from fastapi.testclient import TestClient
|
||||
|
||||
from backend.main import _FALLBACK_MODELS, app
|
||||
|
||||
#: Trimmed-down copy of what api.mistral.ai/v1/models really returns (BUG-044).
|
||||
MISTRAL_LIVE_PAYLOAD = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "mistral-small-latest", "capabilities": {"completion_chat": True, "vision": True}},
|
||||
{"id": "mistral-medium-latest", "capabilities": {"completion_chat": True, "vision": True}},
|
||||
{"id": "mistral-embed", "capabilities": {"completion_chat": False, "vision": False}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_client(tmp_path):
|
||||
@@ -83,6 +93,16 @@ def _login_admin(client):
|
||||
return resp.json()["access_token"]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_declared_caps():
|
||||
"""Provider declarations are cached process-wide: isolate every test."""
|
||||
from backend.provider_capabilities import clear_declared_capabilities
|
||||
|
||||
clear_declared_capabilities()
|
||||
yield
|
||||
clear_declared_capabilities()
|
||||
|
||||
|
||||
# ── Unit tests for the fallback table itself ─────────────────────────────
|
||||
|
||||
|
||||
@@ -339,3 +359,112 @@ class TestListModelsEndpoint:
|
||||
assert xiaomi_hdrs["api-key"] == "fake-key"
|
||||
# Bearer prefix must NOT be in the api-key value
|
||||
assert "Bearer" not in xiaomi_hdrs["api-key"]
|
||||
|
||||
|
||||
class TestDeclaredCapabilities:
|
||||
"""BUG-044 — the provider's own ``capabilities[]`` must reach the UI.
|
||||
|
||||
``GET /api/config/ai-models`` is what fills the capability cache; the picker
|
||||
bubble (``GET /api/ai/model-capabilities``) and the vision gate then read it.
|
||||
Before the fix no Mistral model was ever reported as vision-capable.
|
||||
"""
|
||||
|
||||
def _serve_payload(self, monkeypatch, payload):
|
||||
"""Make the provider models call return ``payload`` verbatim."""
|
||||
import json
|
||||
import urllib.request as global_urllib_mod
|
||||
|
||||
body = json.dumps(payload).encode()
|
||||
|
||||
def fake_Request(url, *args, **kwargs):
|
||||
class FakeResp:
|
||||
def __enter__(self): return self
|
||||
def __exit__(self, *a): pass
|
||||
def read(self): return body
|
||||
|
||||
return FakeResp()
|
||||
|
||||
monkeypatch.setattr(global_urllib_mod, "Request", fake_Request, raising=True)
|
||||
monkeypatch.setattr(
|
||||
global_urllib_mod, "urlopen", lambda req, timeout=None: req, raising=True
|
||||
)
|
||||
|
||||
def _fake_key(self, monkeypatch):
|
||||
"""main.py binds get_ai_key at import: patch that binding too."""
|
||||
import backend.ai as aimod
|
||||
import backend.main as bmain
|
||||
|
||||
monkeypatch.setattr(aimod, "get_ai_key", lambda name: "fake-mistral-key")
|
||||
if hasattr(bmain, "get_ai_key"):
|
||||
monkeypatch.setattr(bmain, "get_ai_key", lambda name: "fake-mistral-key")
|
||||
|
||||
def test_declared_vision_reaches_both_endpoints(self, admin_client, monkeypatch):
|
||||
self._fake_key(monkeypatch)
|
||||
self._serve_payload(monkeypatch, MISTRAL_LIVE_PAYLOAD)
|
||||
|
||||
token = _login_admin(admin_client)
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
listing = admin_client.get(
|
||||
"/api/config/ai-models",
|
||||
params={"provider": "mistral"},
|
||||
headers=headers,
|
||||
)
|
||||
assert listing.status_code == 200, listing.text
|
||||
data = listing.json()
|
||||
assert data["source"] == "live"
|
||||
assert data["capabilities"]["mistral-small-latest"]["vision"] is True
|
||||
assert data["capabilities"]["mistral-medium-latest"]["vision"] is True
|
||||
# The declaration also revokes: mistral-embed is not a chat model.
|
||||
assert data["capabilities"]["mistral-embed"]["chat"] is False
|
||||
assert data["capabilities"]["mistral-embed"]["embeddings"] is True
|
||||
|
||||
for model in ("mistral-small-latest", "mistral-medium-latest"):
|
||||
detail = admin_client.get(
|
||||
"/api/ai/model-capabilities",
|
||||
params={"provider": "mistral", "model": model},
|
||||
headers=headers,
|
||||
)
|
||||
assert detail.status_code == 200, detail.text
|
||||
assert detail.json()["capabilities"]["vision"] is True, model
|
||||
|
||||
def test_text_only_mistral_models_stay_text_only(self, admin_client, monkeypatch):
|
||||
"""mistral-large-latest declares no vision in the real API — keep it so."""
|
||||
self._fake_key(monkeypatch)
|
||||
self._serve_payload(monkeypatch, MISTRAL_LIVE_PAYLOAD)
|
||||
|
||||
token = _login_admin(admin_client)
|
||||
resp = admin_client.get(
|
||||
"/api/config/ai-models",
|
||||
params={"provider": "mistral"},
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
caps = resp.json()["capabilities"]["mistral-large-latest"]
|
||||
assert caps["vision"] is False
|
||||
assert caps["embeddings"] is False
|
||||
|
||||
def test_curated_fallback_covers_mistral_without_any_api_call(
|
||||
self, admin_client, monkeypatch,
|
||||
):
|
||||
"""No API key: the curated list must still answer correctly (offline)."""
|
||||
import backend.ai as aimod
|
||||
import backend.main as bmain
|
||||
|
||||
monkeypatch.setattr(aimod, "get_ai_key", lambda name: None)
|
||||
if hasattr(bmain, "get_ai_key"):
|
||||
monkeypatch.setattr(bmain, "get_ai_key", lambda name: None)
|
||||
|
||||
token = _login_admin(admin_client)
|
||||
resp = admin_client.get(
|
||||
"/api/config/ai-models",
|
||||
params={"provider": "mistral"},
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
assert data["source"] == "fallback"
|
||||
caps = data["capabilities"]
|
||||
assert caps["mistral-small-latest"]["vision"] is True
|
||||
assert caps["mistral-large-latest"]["vision"] is False
|
||||
assert caps["mistral-large-latest"]["embeddings"] is False
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.model_capabilities import (
|
||||
CAPABILITY_KEYS,
|
||||
get_capabilities_for_models,
|
||||
@@ -10,6 +12,16 @@ from backend.model_capabilities import (
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_declared_caps():
|
||||
"""Curated-table tests must not see the process-wide declaration cache."""
|
||||
from backend.provider_capabilities import clear_declared_capabilities
|
||||
|
||||
clear_declared_capabilities()
|
||||
yield
|
||||
clear_declared_capabilities()
|
||||
|
||||
|
||||
class TestCapabilityShape:
|
||||
def test_every_result_has_all_keys(self):
|
||||
for provider, model in [
|
||||
@@ -84,3 +96,85 @@ class TestBatch:
|
||||
)
|
||||
assert result["qwen-max"]["vision"] is False
|
||||
assert result["qwen-vl-max"]["vision"] is True
|
||||
|
||||
|
||||
class TestMistralFamilies:
|
||||
"""BUG-044 — the curated fallback must know the current Mistral families.
|
||||
|
||||
This fallback is what answers when the provider declaration is unavailable
|
||||
(no API key, offline, cold cache), so it has to agree with what
|
||||
``api.mistral.ai/v1/models`` declares today. Before the fix, no Mistral
|
||||
model at all was reported as vision-capable — only the dead ``pixtral``
|
||||
pattern matched.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"mistral-small-latest",
|
||||
"mistral-small-2603",
|
||||
"mistral-medium-latest",
|
||||
"mistral-medium-3.5",
|
||||
"ministral-3b-latest",
|
||||
"ministral-8b-2512",
|
||||
"ministral-14b-latest",
|
||||
"magistral-small-latest",
|
||||
"magistral-medium-latest",
|
||||
"mistral-vibe-cli-latest",
|
||||
"pixtral-12b-2409",
|
||||
],
|
||||
)
|
||||
def test_vision_chat_families(self, model):
|
||||
caps = get_model_capabilities("mistral", model)
|
||||
assert caps["vision"] is True, model
|
||||
assert caps["chat"] is True, model
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"mistral-large-latest",
|
||||
"codestral-latest",
|
||||
"open-mixtral-8x7b",
|
||||
"voxtral-small-latest",
|
||||
"mistral-code-latest",
|
||||
],
|
||||
)
|
||||
def test_text_only_models_are_not_vision(self, model):
|
||||
caps = get_model_capabilities("mistral", model)
|
||||
assert caps["vision"] is False, model
|
||||
assert caps["chat"] is True, model
|
||||
|
||||
def test_ocr_models_are_vision_without_chat(self):
|
||||
caps = get_model_capabilities("mistral", "mistral-ocr-latest")
|
||||
assert caps["vision"] is True
|
||||
assert caps["chat"] is False
|
||||
|
||||
def test_embeddings_only_for_the_embed_models(self):
|
||||
assert get_model_capabilities("mistral", "mistral-embed")["embeddings"] is True
|
||||
# Regression: every Mistral model used to inherit embeddings=True from
|
||||
# the provider default, wrongly branding chat models as embedders.
|
||||
assert get_model_capabilities("mistral", "mistral-large-latest")["embeddings"] is False
|
||||
assert get_model_capabilities("mistral", "some-unknown-mistral")["embeddings"] is False
|
||||
|
||||
def test_current_api_vision_models_are_all_covered(self):
|
||||
"""Every model the API declares vision=true is vision-capable offline.
|
||||
|
||||
List captured from ``GET https://api.mistral.ai/v1/models`` (2026-09-15);
|
||||
the live declaration layer covers renames, this guards the offline path.
|
||||
"""
|
||||
api_vision_models = [
|
||||
"magistral-medium-latest", "magistral-small-latest",
|
||||
"ministral-14b-2512", "ministral-14b-latest",
|
||||
"ministral-3b-2512", "ministral-3b-latest",
|
||||
"ministral-8b-2512", "ministral-8b-latest",
|
||||
"mistral-medium", "mistral-medium-2604", "mistral-medium-3",
|
||||
"mistral-medium-3-5", "mistral-medium-3.5", "mistral-medium-latest",
|
||||
"mistral-ocr-2512", "mistral-ocr-3", "mistral-ocr-3-0",
|
||||
"mistral-ocr-4", "mistral-ocr-4-0", "mistral-ocr-4-1",
|
||||
"mistral-ocr-latest",
|
||||
"mistral-small-2603", "mistral-small-latest",
|
||||
"mistral-vibe-cli-fast", "mistral-vibe-cli-latest",
|
||||
"mistral-vibe-cli-with-tools",
|
||||
]
|
||||
missing = [m for m in api_vision_models if not model_supports_vision("mistral", m)]
|
||||
assert missing == [], f"curated fallback misses vision for: {missing}"
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Tests for provider-declared capabilities + the cache (BUG-044).
|
||||
|
||||
Covers:
|
||||
- payload-shape parsing (Mistral ``capabilities``, OpenRouter ``architecture``);
|
||||
- the in-process cache (TTL, clearing, malformed payloads);
|
||||
- the merge rule: a declaration wins for the flags it mentions, the curated
|
||||
table fills the rest (``backend.model_capabilities``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from backend import provider_capabilities as pc
|
||||
from backend.model_capabilities import CAPABILITY_KEYS, get_model_capabilities
|
||||
|
||||
MISTRAL_PAYLOAD = {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "mistral-small-latest",
|
||||
"capabilities": {
|
||||
"completion_chat": True,
|
||||
"vision": True,
|
||||
"audio_transcription": False,
|
||||
"audio_speech": False,
|
||||
},
|
||||
},
|
||||
{"id": "mistral-embed", "capabilities": {"completion_chat": False, "vision": False}},
|
||||
{"id": "mistral-ocr-latest", "capabilities": {"completion_chat": False, "vision": True}},
|
||||
{
|
||||
"id": "voxtral-mini-latest",
|
||||
"capabilities": {"completion_chat": False, "audio_transcription": True},
|
||||
},
|
||||
{"id": "mistral-new-thing", "capabilities": {"completion_chat": True, "vision": True}},
|
||||
{"id": "mistral-moderation-2603", "capabilities": {"completion_chat": False}},
|
||||
{"id": "no-capabilities-declared"},
|
||||
],
|
||||
}
|
||||
|
||||
OPENROUTER_PAYLOAD = {
|
||||
"data": [
|
||||
{
|
||||
"id": "openai/gpt-4o",
|
||||
"architecture": {
|
||||
"input_modalities": ["text", "image"],
|
||||
"output_modalities": ["text"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "text-only/model",
|
||||
"architecture": {"input_modalities": ["text"], "output_modalities": ["text"]},
|
||||
},
|
||||
{
|
||||
"id": "tts/model",
|
||||
"architecture": {"input_modalities": ["text"], "output_modalities": ["audio"]},
|
||||
},
|
||||
{
|
||||
"id": "whisper/model",
|
||||
"architecture": {"input_modalities": ["audio"], "output_modalities": ["text"]},
|
||||
},
|
||||
{
|
||||
"id": "video/model",
|
||||
"architecture": {"input_modalities": ["text"], "output_modalities": ["video"]},
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_cache():
|
||||
"""The capability cache is process-wide: never leak between tests."""
|
||||
pc.clear_declared_capabilities()
|
||||
yield
|
||||
pc.clear_declared_capabilities()
|
||||
|
||||
|
||||
class TestParsing:
|
||||
def test_mistral_capability_flags_are_mapped(self):
|
||||
declared = pc.parse_declared_capabilities(MISTRAL_PAYLOAD["data"][0])
|
||||
assert declared == {
|
||||
"chat": True,
|
||||
"vision": True,
|
||||
"audio_transcription": False,
|
||||
"audio_speech": False,
|
||||
}
|
||||
assert set(declared) <= set(CAPABILITY_KEYS)
|
||||
|
||||
def test_mistral_ocr_is_vision_without_chat(self):
|
||||
declared = pc.parse_declared_capabilities(MISTRAL_PAYLOAD["data"][2])
|
||||
assert declared == {"chat": False, "vision": True}
|
||||
|
||||
def test_realtime_transcription_counts_as_transcription(self):
|
||||
entry = {
|
||||
"id": "model",
|
||||
"capabilities": {"completion_chat": False, "audio_transcription_realtime": True},
|
||||
}
|
||||
assert pc.parse_declared_capabilities(entry) == {
|
||||
"chat": False,
|
||||
"audio_transcription": True,
|
||||
}
|
||||
|
||||
def test_openrouter_modalities_are_mapped(self):
|
||||
entries = {e["id"]: e for e in OPENROUTER_PAYLOAD["data"]}
|
||||
assert pc.parse_declared_capabilities(entries["openai/gpt-4o"]) == {
|
||||
"chat": True,
|
||||
"vision": True,
|
||||
"images": False,
|
||||
"audio_transcription": False,
|
||||
"audio_speech": False,
|
||||
"video": False,
|
||||
}
|
||||
assert pc.parse_declared_capabilities(entries["tts/model"])["audio_speech"] is True
|
||||
assert pc.parse_declared_capabilities(entries["whisper/model"])["audio_transcription"] is True
|
||||
assert pc.parse_declared_capabilities(entries["video/model"])["video"] is True
|
||||
text_only = pc.parse_declared_capabilities(entries["text-only/model"])
|
||||
assert text_only is not None and text_only["vision"] is False
|
||||
|
||||
def test_undeclared_entries_return_none(self):
|
||||
for entry in [None, "not-a-dict", {}, {"architecture": {}}, {"capabilities": {}}]:
|
||||
assert pc.parse_declared_capabilities(entry) is None
|
||||
|
||||
|
||||
class TestCache:
|
||||
def test_remember_returns_cached_model_count(self):
|
||||
# 7 entries, 1 without any declaration.
|
||||
assert pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD) == 6
|
||||
assert pc.cache_info()["mistral"]["models"] == 6
|
||||
|
||||
def test_unknown_model_returns_none(self):
|
||||
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
|
||||
assert pc.get_declared_capabilities("mistral", "not-in-payload") is None
|
||||
|
||||
def test_unknown_provider_returns_none(self):
|
||||
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
|
||||
assert pc.get_declared_capabilities("openrouter", "mistral-small-latest") is None
|
||||
|
||||
def test_payload_can_be_a_bare_list(self):
|
||||
assert pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD["data"]) == 6
|
||||
|
||||
def test_shape_that_declares_nothing_is_not_cached(self):
|
||||
"""Gemini-shaped payloads (``{"models": [...]}``) declare nothing."""
|
||||
gemini_like = {"models": [{"name": "models/gemini-2.0-flash"}]}
|
||||
assert pc.remember_declared_capabilities("gemini", gemini_like) == 0
|
||||
assert "gemini" not in pc.cache_info()
|
||||
|
||||
def test_gemini_models_prefix_is_normalised(self):
|
||||
payload = [{"name": "models/gemini-x", "capabilities": {"completion_chat": True}}]
|
||||
pc.remember_declared_capabilities("gemini", payload)
|
||||
assert pc.get_declared_capabilities("gemini", "gemini-x") == {"chat": True}
|
||||
assert pc.get_declared_capabilities("gemini", "models/gemini-x") == {"chat": True}
|
||||
|
||||
def test_snapshot_expires_after_ttl(self, monkeypatch):
|
||||
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
|
||||
monkeypatch.setattr(pc, "TTL_SECONDS", 0.01)
|
||||
time.sleep(0.05)
|
||||
assert pc.get_declared_capabilities("mistral", "mistral-small-latest") is None
|
||||
assert pc.cache_info()["mistral"]["expired"] is True
|
||||
|
||||
def test_clear_one_provider_only(self):
|
||||
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
|
||||
pc.remember_declared_capabilities("openrouter", OPENROUTER_PAYLOAD)
|
||||
pc.clear_declared_capabilities("mistral")
|
||||
assert pc.get_declared_capabilities("mistral", "mistral-small-latest") is None
|
||||
assert pc.get_declared_capabilities("openrouter", "openai/gpt-4o") is not None
|
||||
|
||||
|
||||
class TestMergeWithCuratedTable:
|
||||
def test_declaration_fixes_a_model_the_curated_table_never_heard_of(self):
|
||||
model = "mistral-new-thing"
|
||||
assert get_model_capabilities("mistral", model)["vision"] is False # curated alone
|
||||
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
|
||||
caps = get_model_capabilities("mistral", model)
|
||||
assert caps["vision"] is True
|
||||
assert caps["chat"] is True
|
||||
assert set(caps) == set(CAPABILITY_KEYS)
|
||||
|
||||
def test_declaration_revokes_a_wrong_curated_guess(self):
|
||||
"""voxtral is audio-only: the curated default called it a chat model."""
|
||||
assert get_model_capabilities("mistral", "voxtral-mini-latest")["chat"] is True
|
||||
assert get_model_capabilities("mistral", "voxtral-mini-latest")["audio_transcription"] is False
|
||||
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
|
||||
caps = get_model_capabilities("mistral", "voxtral-mini-latest")
|
||||
assert caps["chat"] is False
|
||||
assert caps["audio_transcription"] is True
|
||||
|
||||
def test_curated_table_fills_flags_the_provider_stays_silent_about(self):
|
||||
"""Mistral never declares ``embedding``: only the absence of chat."""
|
||||
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
|
||||
caps = get_model_capabilities("mistral", "mistral-embed")
|
||||
assert caps["chat"] is False
|
||||
assert caps["embeddings"] is True
|
||||
|
||||
def test_openrouter_vision_comes_from_the_declaration(self):
|
||||
pc.remember_declared_capabilities("openrouter", OPENROUTER_PAYLOAD)
|
||||
assert get_model_capabilities("openrouter", "openai/gpt-4o")["vision"] is True
|
||||
assert get_model_capabilities("openrouter", "text-only/model")["vision"] is False
|
||||
|
||||
def test_clearing_the_cache_restores_the_curated_answer(self):
|
||||
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
|
||||
assert get_model_capabilities("mistral", "mistral-new-thing")["vision"] is True
|
||||
pc.clear_declared_capabilities()
|
||||
assert get_model_capabilities("mistral", "mistral-new-thing")["vision"] is False
|
||||
Reference in New Issue
Block a user