feat(ai): ecosysteme d'outils phase 2 - recherche a cle, cache/retry, Playwright, crawl, Gitea/GitHub, documents (#92)

backend/tools/web.py

@@ -8,6 +8,16 @@ Phase 1 of the documented web-toolset roadmap:
   answering « je n'ai pas accès à internet ».
 * ``fetch_url``   — retrieve a public web page and return readable text.
 
+Phase 2 (#92) additions:
+
+* keyed providers — Tavily, Brave Search, SerpAPI and Exa are used first when
+  their API key is configured (env, injected by Infisical in production);
+* SQLite cache — search/fetch results are cached with a TTL
+  (:mod:`backend.tools.webcache`);
+* retry with backoff — transient network errors get one extra attempt;
+* dynamic rendering — ``fetch_url(render=True)`` uses an isolated Playwright
+  worker (optional dependency, graceful degradation).
+
 All are READ-risk tools (no confirmation), rate-limited through the shared
 registry, SSRF-guarded (scheme + private-address rejection), and size-capped.
 
@@ -16,6 +26,14 @@ Configuration (environment):
 * ``OBSIGATE_WEB_TIMEOUT``   — seconds, default 10
 * ``OBSIGATE_WEB_FALLBACK``  — ``0``/``false`` disables the keyless HTML
   fallbacks (SearXNG only), default enabled
+* ``OBSIGATE_TAVILY_API_KEY`` / ``OBSIGATE_BRAVE_API_KEY`` /
+  ``OBSIGATE_SERPAPI_API_KEY`` / ``OBSIGATE_EXA_API_KEY`` — optional keyed
+  providers, tried before SearXNG when set
+* ``OBSIGATE_WEB_PROVIDERS`` — optional comma-separated provider order
+  (e.g. ``brave,searxng``); keyed providers without a key are skipped
+* ``OBSIGATE_WEB_RETRY``     — extra attempts for transient network errors
+  (default 1)
+* ``OBSIGATE_WEB_CACHE_TTL`` — cache TTL seconds, ``0`` disables (default 900)
 """
 
 from __future__ import annotations
@@ -28,12 +46,14 @@ import logging
 import os
 import re
 import socket
+import time
 from collections.abc import Callable
 from typing import Any
 from urllib.parse import parse_qs, urlparse
 
 import httpx
 
+from backend.tools import webcache
 from backend.tools.context import ToolError, ToolRisk, ToolScope
 from backend.tools.registry import tool
 from backend.tools.schemas import FetchUrlInput, WebSearchInput
@@ -48,6 +68,7 @@ WEB_FALLBACK_ENABLED = os.environ.get("OBSIGATE_WEB_FALLBACK", "1").strip().lowe
     "no",
     "off",
 }
+WEB_RETRY_ATTEMPTS = int(os.environ.get("OBSIGATE_WEB_RETRY", "1"))
 USER_AGENT = "ObsiGateAssistant/1.0 (+self-hosted vault AI)"
 # Search engines reject non-browser agents on their public HTML endpoints.
 BROWSER_UA = (
@@ -163,6 +184,115 @@ def _result(
     }
 
 
+def _with_retry(call: Callable[[], Any]) -> Any:
+    """Run *call* with one extra attempt on transient network errors.
+
+    House-made backoff (the roadmap's « tenacity ou boucle maison »): DNS
+    blips and rate-limit hiccups are the common failure mode, and a single
+    retry keeps the fallback chain from being consumed too early.
+    """
+    for attempt in range(1 + max(0, WEB_RETRY_ATTEMPTS)):
+        try:
+            return call()
+        except httpx.TransportError:
+            if attempt >= max(0, WEB_RETRY_ATTEMPTS):
+                raise
+            time.sleep(0.2 * (attempt + 1))
+    raise RuntimeError("unreachable")  # pragma: no cover
+
+
+def _env_key(name: str) -> str:
+    """Read an API key from the environment (Infisical-injected in prod)."""
+    return os.environ.get(name, "").strip()
+
+
+def _search_tavily(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]:
+    """Tavily Search API (agent-oriented results, key required)."""
+    resp = httpx.post(
+        "https://api.tavily.com/search",
+        json={
+            "api_key": _env_key("OBSIGATE_TAVILY_API_KEY"),
+            "query": query,
+            "max_results": params.max_results,
+            "search_depth": "basic",
+            "include_answer": False,
+        },
+        headers={"User-Agent": USER_AGENT},
+        timeout=WEB_TIMEOUT,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    return [
+        _result(item.get("title") or "", item.get("url") or "", item.get("content") or "")
+        for item in (data.get("results") or [])
+    ], []
+
+
+def _search_brave(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]:
+    """Brave Search API (key required)."""
+    resp = httpx.get(
+        "https://api.search.brave.com/res/v1/web/search",
+        params={"q": query, "count": params.max_results, "safesearch": "moderate"},
+        headers={
+            "X-Subscription-Id": _env_key("OBSIGATE_BRAVE_API_KEY"),
+            "Accept": "application/json",
+            "User-Agent": USER_AGENT,
+        },
+        timeout=WEB_TIMEOUT,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    return [
+        _result(item.get("title") or "", item.get("url") or "", item.get("description") or "")
+        for item in ((data.get("web") or {}).get("results") or [])
+    ], []
+
+
+def _search_serpapi(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]:
+    """SerpAPI (Google SERP, key required)."""
+    resp = httpx.get(
+        "https://serpapi.com/search",
+        params={"q": query, "api_key": _env_key("OBSIGATE_SERPAPI_API_KEY"),
+                "num": params.max_results},
+        headers={"User-Agent": USER_AGENT},
+        timeout=WEB_TIMEOUT,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    return [
+        _result(item.get("title") or "", item.get("link") or "", item.get("snippet") or "")
+        for item in (data.get("organic_results") or [])
+    ], []
+
+
+def _search_exa(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]:
+    """Exa neural search (key required)."""
+    resp = httpx.post(
+        "https://api.exa.ai/search",
+        json={"query": query, "numResults": params.max_results},
+        headers={
+            "x-api-key": _env_key("OBSIGATE_EXA_API_KEY"),
+            "User-Agent": USER_AGENT,
+        },
+        timeout=WEB_TIMEOUT,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    return [
+        _result(item.get("title") or "", item.get("url") or "", (item.get("text") or "")[:600])
+        for item in (data.get("results") or [])
+    ], []
+
+
+# Keyed providers: name -> (implementation, API key env var)
+_KEYED_PROVIDERS: dict[str, tuple[_Provider, str]] = {
+    "tavily": (_search_tavily, "OBSIGATE_TAVILY_API_KEY"),
+    "brave": (_search_brave, "OBSIGATE_BRAVE_API_KEY"),
+    "serpapi": (_search_serpapi, "OBSIGATE_SERPAPI_API_KEY"),
+    "exa": (_search_exa, "OBSIGATE_EXA_API_KEY"),
+}
+
+
 def _search_searxng(
     query: str, params: WebSearchInput
 ) -> tuple[list[dict[str, Any]], list[str]]:
@@ -288,8 +418,22 @@ _Provider = Callable[[str, WebSearchInput], "tuple[list[dict[str, Any]], list[st
 
 
 def _provider_chain() -> list[tuple[str, _Provider]]:
-    """Ordered providers: self-hosted meta-search first, then keyless fallbacks."""
-    chain: list[tuple[str, _Provider]] = [("searxng", _search_searxng)]
+    """Ordered providers: keyed APIs first, then self-hosted, then keyless.
+
+    ``OBSIGATE_WEB_PROVIDERS`` (comma-separated) overrides the default order;
+    unknown names are ignored and keyed providers without their key are skipped.
+    """
+    chain: list[tuple[str, _Provider]] = []
+    configured = [
+        name.strip().lower()
+        for name in os.environ.get("OBSIGATE_WEB_PROVIDERS", "").split(",")
+        if name.strip()
+    ]
+    for name in configured or list(_KEYED_PROVIDERS):
+        entry = _KEYED_PROVIDERS.get(name)
+        if entry and _env_key(entry[1]):
+            chain.append((name, entry[0]))
+    chain.append(("searxng", _search_searxng))
     if WEB_FALLBACK_ENABLED:
         chain.append(("duckduckgo", _search_duckduckgo))
         chain.append(("bing", _search_bing))
@@ -313,6 +457,17 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
     if not query:
         raise ToolError("Requête vide", code="invalid_arguments")
 
+    key = webcache.cache_key("search", {
+        "q": query,
+        "max_results": params.max_results,
+        "category": params.category,
+        "language": params.language,
+        "page": params.page,
+    })
+    cached = webcache.cache_get(key)
+    if cached is not None:
+        return {**cached, "cached": True}
+
     attempts: list[str] = []
     unresponsive: list[str] = []
     reachable = False
@@ -320,8 +475,12 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
 
     for name, provider in _provider_chain():
         attempts.append(name)
+
+        def _attempt(p: _Provider = provider) -> tuple[list[dict[str, Any]], list[str]]:
+            return p(query, params)
+
         try:
-            results, engines = provider(query, params)
+            results, engines = _with_retry(_attempt)
         except (httpx.HTTPError, ValueError, AttributeError) as e:
             logger.warning("web_search provider %s failed: %s", name, e)
             last_error = e
@@ -338,6 +497,7 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
             }
             if unresponsive:
                 payload["unresponsive_engines"] = unresponsive[:8]
+            webcache.cache_set(key, payload)
             return payload
 
     if not reachable:
@@ -378,6 +538,20 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
 def fetch_url(ctx, params: FetchUrlInput) -> dict[str, Any]:
     """Retrieve one page, guard against SSRF, and extract its text."""
     url = _assert_public_http_url(params.url.strip())
+    key = webcache.cache_key("fetch", {"url": url, "render": params.render})
+    cached = webcache.cache_get(key)
+    if cached is not None:
+        return {**cached, "cached": True}
+
+    if params.render:
+        # Dynamic pages (SPA/React): delegated to the isolated Playwright
+        # worker; the browser dependency stays optional (graceful error).
+        from backend.tools.webrender import render_page
+
+        payload = render_page(url)
+        webcache.cache_set(key, payload)
+        return payload
+
     try:
         # Follow redirects manually so every hop is re-checked against the
         # private-address SSRF guard (a public page can redirect to 127.0.0.1).
@@ -414,10 +588,12 @@ def fetch_url(ctx, params: FetchUrlInput) -> dict[str, Any]:
     title_match = re.search(r"<title[^>]*>(.*?)</title>", raw, re.IGNORECASE | re.DOTALL)
     title = html_lib.unescape(title_match.group(1)).strip()[:300] if title_match else ""
     text = _html_to_text(raw)[:MAX_TEXT_CHARS]
-    return {
+    payload = {
         "url": str(resp.url),
         "status": resp.status_code,
         "title": title,
         "text": text,
         "truncated": len(raw) > MAX_TEXT_CHARS,
     }
+    webcache.cache_set(key, payload)
+    return payload