fix(ai): BUG-044 capacités des modèles lues chez le fournisseur (Mistral vision)
CI / lint (push) Successful in 1m20s
CI / security (push) Successful in 53s
CI / test (push) Successful in 2m27s
CI / build (push) Successful in 50s
CI / e2e (push) Successful in 10m48s

Le panneau de modèle par défaut et la bulle ⓘ n'affichaient aucun modèle Mistral
« Vision capable » alors que GET api.mistral.ai/v1/models en déclare 28 : la table
de capacités était entièrement statique et aucun de ses motifs ne correspondait aux
familles Mistral actuelles (seul `pixtral`, retiré de l'API, les matchait).

- backend/provider_capabilities.py (nouveau) : capacités déclarées par le
  fournisseur (Mistral `capabilities`, OpenRouter `architecture`), détectées par
  la forme du payload, snapshot en cache process-wide (TTL 30 min, surchargeable
  par AI_CAPABILITIES_TTL_SECONDS) rempli par GET /api/config/ai-models.
- backend/model_capabilities.py : une déclaration prime sur la table statique
  pour chaque drapeau mentionné ; la table ne comble que le reste (Mistral ne
  déclare jamais `embedding`). Table corrigée pour le repli hors ligne : familles
  vision Mistral (ministral, magistral, mistral-small, mistral-medium,
  mistral-vibe-cli, labs-leanstral), mistral-ocr = vision sans chat, et défaut du
  fournisseur Mistral sans `embeddings` (mistral-large / codestral n'étaient plus
  des « embedders »).
- backend/ai_routes.py : GET /api/ai/model-capabilities reste sans appel réseau
  (cache froid → table statique).
- Tests : tests/test_provider_capabilities.py (nouveau), TestMistralFamilies et
  TestDeclaredCapabilities (bout en bout via l'API).
- Docs : CHANGELOG [Unreleased], registre + journal ISSUES_TODOLIST,
  fiche docs/features/ai-provider-picker.md (§L).

Vérifié : 28/28 modèles vision déclarés par Mistral détectés (0 avant), 0 écart
dans les deux sens ; pytest 1007 passed / 6 skipped ; ruff 0 (backend) ; mypy 0 ;
tests frontend unit 9/9 + IA 66/66 + validate-imports 37 modules ; instance de test
reconstruite et vérifiée sur http://localhost:2020.
This commit is contained in:
2026-09-15 09:26:31 -04:00
parent 140a8f6efe
commit b69cb9b0f8
10 changed files with 723 additions and 23 deletions
+129
View File
@@ -14,6 +14,16 @@ from fastapi.testclient import TestClient
from backend.main import _FALLBACK_MODELS, app
#: Trimmed-down copy of what api.mistral.ai/v1/models really returns (BUG-044).
MISTRAL_LIVE_PAYLOAD = {
"object": "list",
"data": [
{"id": "mistral-small-latest", "capabilities": {"completion_chat": True, "vision": True}},
{"id": "mistral-medium-latest", "capabilities": {"completion_chat": True, "vision": True}},
{"id": "mistral-embed", "capabilities": {"completion_chat": False, "vision": False}},
],
}
@pytest.fixture
def admin_client(tmp_path):
@@ -83,6 +93,16 @@ def _login_admin(client):
return resp.json()["access_token"]
@pytest.fixture(autouse=True)
def _no_declared_caps():
"""Provider declarations are cached process-wide: isolate every test."""
from backend.provider_capabilities import clear_declared_capabilities
clear_declared_capabilities()
yield
clear_declared_capabilities()
# ── Unit tests for the fallback table itself ─────────────────────────────
@@ -339,3 +359,112 @@ class TestListModelsEndpoint:
assert xiaomi_hdrs["api-key"] == "fake-key"
# Bearer prefix must NOT be in the api-key value
assert "Bearer" not in xiaomi_hdrs["api-key"]
class TestDeclaredCapabilities:
"""BUG-044 — the provider's own ``capabilities[]`` must reach the UI.
``GET /api/config/ai-models`` is what fills the capability cache; the picker
bubble (``GET /api/ai/model-capabilities``) and the vision gate then read it.
Before the fix no Mistral model was ever reported as vision-capable.
"""
def _serve_payload(self, monkeypatch, payload):
"""Make the provider models call return ``payload`` verbatim."""
import json
import urllib.request as global_urllib_mod
body = json.dumps(payload).encode()
def fake_Request(url, *args, **kwargs):
class FakeResp:
def __enter__(self): return self
def __exit__(self, *a): pass
def read(self): return body
return FakeResp()
monkeypatch.setattr(global_urllib_mod, "Request", fake_Request, raising=True)
monkeypatch.setattr(
global_urllib_mod, "urlopen", lambda req, timeout=None: req, raising=True
)
def _fake_key(self, monkeypatch):
"""main.py binds get_ai_key at import: patch that binding too."""
import backend.ai as aimod
import backend.main as bmain
monkeypatch.setattr(aimod, "get_ai_key", lambda name: "fake-mistral-key")
if hasattr(bmain, "get_ai_key"):
monkeypatch.setattr(bmain, "get_ai_key", lambda name: "fake-mistral-key")
def test_declared_vision_reaches_both_endpoints(self, admin_client, monkeypatch):
self._fake_key(monkeypatch)
self._serve_payload(monkeypatch, MISTRAL_LIVE_PAYLOAD)
token = _login_admin(admin_client)
headers = {"Authorization": f"Bearer {token}"}
listing = admin_client.get(
"/api/config/ai-models",
params={"provider": "mistral"},
headers=headers,
)
assert listing.status_code == 200, listing.text
data = listing.json()
assert data["source"] == "live"
assert data["capabilities"]["mistral-small-latest"]["vision"] is True
assert data["capabilities"]["mistral-medium-latest"]["vision"] is True
# The declaration also revokes: mistral-embed is not a chat model.
assert data["capabilities"]["mistral-embed"]["chat"] is False
assert data["capabilities"]["mistral-embed"]["embeddings"] is True
for model in ("mistral-small-latest", "mistral-medium-latest"):
detail = admin_client.get(
"/api/ai/model-capabilities",
params={"provider": "mistral", "model": model},
headers=headers,
)
assert detail.status_code == 200, detail.text
assert detail.json()["capabilities"]["vision"] is True, model
def test_text_only_mistral_models_stay_text_only(self, admin_client, monkeypatch):
"""mistral-large-latest declares no vision in the real API — keep it so."""
self._fake_key(monkeypatch)
self._serve_payload(monkeypatch, MISTRAL_LIVE_PAYLOAD)
token = _login_admin(admin_client)
resp = admin_client.get(
"/api/config/ai-models",
params={"provider": "mistral"},
headers={"Authorization": f"Bearer {token}"},
)
assert resp.status_code == 200, resp.text
caps = resp.json()["capabilities"]["mistral-large-latest"]
assert caps["vision"] is False
assert caps["embeddings"] is False
def test_curated_fallback_covers_mistral_without_any_api_call(
self, admin_client, monkeypatch,
):
"""No API key: the curated list must still answer correctly (offline)."""
import backend.ai as aimod
import backend.main as bmain
monkeypatch.setattr(aimod, "get_ai_key", lambda name: None)
if hasattr(bmain, "get_ai_key"):
monkeypatch.setattr(bmain, "get_ai_key", lambda name: None)
token = _login_admin(admin_client)
resp = admin_client.get(
"/api/config/ai-models",
params={"provider": "mistral"},
headers={"Authorization": f"Bearer {token}"},
)
assert resp.status_code == 200, resp.text
data = resp.json()
assert data["source"] == "fallback"
caps = data["capabilities"]
assert caps["mistral-small-latest"]["vision"] is True
assert caps["mistral-large-latest"]["vision"] is False
assert caps["mistral-large-latest"]["embeddings"] is False
+94
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
import pytest
from backend.model_capabilities import (
CAPABILITY_KEYS,
get_capabilities_for_models,
@@ -10,6 +12,16 @@ from backend.model_capabilities import (
)
@pytest.fixture(autouse=True)
def _no_declared_caps():
"""Curated-table tests must not see the process-wide declaration cache."""
from backend.provider_capabilities import clear_declared_capabilities
clear_declared_capabilities()
yield
clear_declared_capabilities()
class TestCapabilityShape:
def test_every_result_has_all_keys(self):
for provider, model in [
@@ -84,3 +96,85 @@ class TestBatch:
)
assert result["qwen-max"]["vision"] is False
assert result["qwen-vl-max"]["vision"] is True
class TestMistralFamilies:
"""BUG-044 — the curated fallback must know the current Mistral families.
This fallback is what answers when the provider declaration is unavailable
(no API key, offline, cold cache), so it has to agree with what
``api.mistral.ai/v1/models`` declares today. Before the fix, no Mistral
model at all was reported as vision-capable — only the dead ``pixtral``
pattern matched.
"""
@pytest.mark.parametrize(
"model",
[
"mistral-small-latest",
"mistral-small-2603",
"mistral-medium-latest",
"mistral-medium-3.5",
"ministral-3b-latest",
"ministral-8b-2512",
"ministral-14b-latest",
"magistral-small-latest",
"magistral-medium-latest",
"mistral-vibe-cli-latest",
"pixtral-12b-2409",
],
)
def test_vision_chat_families(self, model):
caps = get_model_capabilities("mistral", model)
assert caps["vision"] is True, model
assert caps["chat"] is True, model
@pytest.mark.parametrize(
"model",
[
"mistral-large-latest",
"codestral-latest",
"open-mixtral-8x7b",
"voxtral-small-latest",
"mistral-code-latest",
],
)
def test_text_only_models_are_not_vision(self, model):
caps = get_model_capabilities("mistral", model)
assert caps["vision"] is False, model
assert caps["chat"] is True, model
def test_ocr_models_are_vision_without_chat(self):
caps = get_model_capabilities("mistral", "mistral-ocr-latest")
assert caps["vision"] is True
assert caps["chat"] is False
def test_embeddings_only_for_the_embed_models(self):
assert get_model_capabilities("mistral", "mistral-embed")["embeddings"] is True
# Regression: every Mistral model used to inherit embeddings=True from
# the provider default, wrongly branding chat models as embedders.
assert get_model_capabilities("mistral", "mistral-large-latest")["embeddings"] is False
assert get_model_capabilities("mistral", "some-unknown-mistral")["embeddings"] is False
def test_current_api_vision_models_are_all_covered(self):
"""Every model the API declares vision=true is vision-capable offline.
List captured from ``GET https://api.mistral.ai/v1/models`` (2026-09-15);
the live declaration layer covers renames, this guards the offline path.
"""
api_vision_models = [
"magistral-medium-latest", "magistral-small-latest",
"ministral-14b-2512", "ministral-14b-latest",
"ministral-3b-2512", "ministral-3b-latest",
"ministral-8b-2512", "ministral-8b-latest",
"mistral-medium", "mistral-medium-2604", "mistral-medium-3",
"mistral-medium-3-5", "mistral-medium-3.5", "mistral-medium-latest",
"mistral-ocr-2512", "mistral-ocr-3", "mistral-ocr-3-0",
"mistral-ocr-4", "mistral-ocr-4-0", "mistral-ocr-4-1",
"mistral-ocr-latest",
"mistral-small-2603", "mistral-small-latest",
"mistral-vibe-cli-fast", "mistral-vibe-cli-latest",
"mistral-vibe-cli-with-tools",
]
missing = [m for m in api_vision_models if not model_supports_vision("mistral", m)]
assert missing == [], f"curated fallback misses vision for: {missing}"
+205
View File
@@ -0,0 +1,205 @@
"""Tests for provider-declared capabilities + the cache (BUG-044).
Covers:
- payload-shape parsing (Mistral ``capabilities``, OpenRouter ``architecture``);
- the in-process cache (TTL, clearing, malformed payloads);
- the merge rule: a declaration wins for the flags it mentions, the curated
table fills the rest (``backend.model_capabilities``).
"""
from __future__ import annotations
import time
import pytest
from backend import provider_capabilities as pc
from backend.model_capabilities import CAPABILITY_KEYS, get_model_capabilities
MISTRAL_PAYLOAD = {
"object": "list",
"data": [
{
"id": "mistral-small-latest",
"capabilities": {
"completion_chat": True,
"vision": True,
"audio_transcription": False,
"audio_speech": False,
},
},
{"id": "mistral-embed", "capabilities": {"completion_chat": False, "vision": False}},
{"id": "mistral-ocr-latest", "capabilities": {"completion_chat": False, "vision": True}},
{
"id": "voxtral-mini-latest",
"capabilities": {"completion_chat": False, "audio_transcription": True},
},
{"id": "mistral-new-thing", "capabilities": {"completion_chat": True, "vision": True}},
{"id": "mistral-moderation-2603", "capabilities": {"completion_chat": False}},
{"id": "no-capabilities-declared"},
],
}
OPENROUTER_PAYLOAD = {
"data": [
{
"id": "openai/gpt-4o",
"architecture": {
"input_modalities": ["text", "image"],
"output_modalities": ["text"],
},
},
{
"id": "text-only/model",
"architecture": {"input_modalities": ["text"], "output_modalities": ["text"]},
},
{
"id": "tts/model",
"architecture": {"input_modalities": ["text"], "output_modalities": ["audio"]},
},
{
"id": "whisper/model",
"architecture": {"input_modalities": ["audio"], "output_modalities": ["text"]},
},
{
"id": "video/model",
"architecture": {"input_modalities": ["text"], "output_modalities": ["video"]},
},
]
}
@pytest.fixture(autouse=True)
def _clean_cache():
"""The capability cache is process-wide: never leak between tests."""
pc.clear_declared_capabilities()
yield
pc.clear_declared_capabilities()
class TestParsing:
def test_mistral_capability_flags_are_mapped(self):
declared = pc.parse_declared_capabilities(MISTRAL_PAYLOAD["data"][0])
assert declared == {
"chat": True,
"vision": True,
"audio_transcription": False,
"audio_speech": False,
}
assert set(declared) <= set(CAPABILITY_KEYS)
def test_mistral_ocr_is_vision_without_chat(self):
declared = pc.parse_declared_capabilities(MISTRAL_PAYLOAD["data"][2])
assert declared == {"chat": False, "vision": True}
def test_realtime_transcription_counts_as_transcription(self):
entry = {
"id": "model",
"capabilities": {"completion_chat": False, "audio_transcription_realtime": True},
}
assert pc.parse_declared_capabilities(entry) == {
"chat": False,
"audio_transcription": True,
}
def test_openrouter_modalities_are_mapped(self):
entries = {e["id"]: e for e in OPENROUTER_PAYLOAD["data"]}
assert pc.parse_declared_capabilities(entries["openai/gpt-4o"]) == {
"chat": True,
"vision": True,
"images": False,
"audio_transcription": False,
"audio_speech": False,
"video": False,
}
assert pc.parse_declared_capabilities(entries["tts/model"])["audio_speech"] is True
assert pc.parse_declared_capabilities(entries["whisper/model"])["audio_transcription"] is True
assert pc.parse_declared_capabilities(entries["video/model"])["video"] is True
text_only = pc.parse_declared_capabilities(entries["text-only/model"])
assert text_only is not None and text_only["vision"] is False
def test_undeclared_entries_return_none(self):
for entry in [None, "not-a-dict", {}, {"architecture": {}}, {"capabilities": {}}]:
assert pc.parse_declared_capabilities(entry) is None
class TestCache:
def test_remember_returns_cached_model_count(self):
# 7 entries, 1 without any declaration.
assert pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD) == 6
assert pc.cache_info()["mistral"]["models"] == 6
def test_unknown_model_returns_none(self):
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
assert pc.get_declared_capabilities("mistral", "not-in-payload") is None
def test_unknown_provider_returns_none(self):
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
assert pc.get_declared_capabilities("openrouter", "mistral-small-latest") is None
def test_payload_can_be_a_bare_list(self):
assert pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD["data"]) == 6
def test_shape_that_declares_nothing_is_not_cached(self):
"""Gemini-shaped payloads (``{"models": [...]}``) declare nothing."""
gemini_like = {"models": [{"name": "models/gemini-2.0-flash"}]}
assert pc.remember_declared_capabilities("gemini", gemini_like) == 0
assert "gemini" not in pc.cache_info()
def test_gemini_models_prefix_is_normalised(self):
payload = [{"name": "models/gemini-x", "capabilities": {"completion_chat": True}}]
pc.remember_declared_capabilities("gemini", payload)
assert pc.get_declared_capabilities("gemini", "gemini-x") == {"chat": True}
assert pc.get_declared_capabilities("gemini", "models/gemini-x") == {"chat": True}
def test_snapshot_expires_after_ttl(self, monkeypatch):
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
monkeypatch.setattr(pc, "TTL_SECONDS", 0.01)
time.sleep(0.05)
assert pc.get_declared_capabilities("mistral", "mistral-small-latest") is None
assert pc.cache_info()["mistral"]["expired"] is True
def test_clear_one_provider_only(self):
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
pc.remember_declared_capabilities("openrouter", OPENROUTER_PAYLOAD)
pc.clear_declared_capabilities("mistral")
assert pc.get_declared_capabilities("mistral", "mistral-small-latest") is None
assert pc.get_declared_capabilities("openrouter", "openai/gpt-4o") is not None
class TestMergeWithCuratedTable:
def test_declaration_fixes_a_model_the_curated_table_never_heard_of(self):
model = "mistral-new-thing"
assert get_model_capabilities("mistral", model)["vision"] is False # curated alone
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
caps = get_model_capabilities("mistral", model)
assert caps["vision"] is True
assert caps["chat"] is True
assert set(caps) == set(CAPABILITY_KEYS)
def test_declaration_revokes_a_wrong_curated_guess(self):
"""voxtral is audio-only: the curated default called it a chat model."""
assert get_model_capabilities("mistral", "voxtral-mini-latest")["chat"] is True
assert get_model_capabilities("mistral", "voxtral-mini-latest")["audio_transcription"] is False
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
caps = get_model_capabilities("mistral", "voxtral-mini-latest")
assert caps["chat"] is False
assert caps["audio_transcription"] is True
def test_curated_table_fills_flags_the_provider_stays_silent_about(self):
"""Mistral never declares ``embedding``: only the absence of chat."""
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
caps = get_model_capabilities("mistral", "mistral-embed")
assert caps["chat"] is False
assert caps["embeddings"] is True
def test_openrouter_vision_comes_from_the_declaration(self):
pc.remember_declared_capabilities("openrouter", OPENROUTER_PAYLOAD)
assert get_model_capabilities("openrouter", "openai/gpt-4o")["vision"] is True
assert get_model_capabilities("openrouter", "text-only/model")["vision"] is False
def test_clearing_the_cache_restores_the_curated_answer(self):
pc.remember_declared_capabilities("mistral", MISTRAL_PAYLOAD)
assert get_model_capabilities("mistral", "mistral-new-thing")["vision"] is True
pc.clear_declared_capabilities()
assert get_model_capabilities("mistral", "mistral-new-thing")["vision"] is False