fix(ai): chaine de repli web_search (SearXNG -> DuckDuckGo -> Bing, BUG-051)

backend/tools/web.py

@@ -2,27 +2,35 @@
 
 Phase 1 of the documented web-toolset roadmap:
 
-* ``web_search``  — query the self-hosted SearXNG instance (no API key).
+* ``web_search``  — query the self-hosted SearXNG instance (no API key) and,
+  when it returns nothing, fall back to keyless HTML providers (DuckDuckGo,
+  then Bing) so a dead meta-search instance never leaves the assistant
+  answering « je n'ai pas accès à internet ».
 * ``fetch_url``   — retrieve a public web page and return readable text.
 
-Both are READ-risk tools (no confirmation), rate-limited through the shared
+All are READ-risk tools (no confirmation), rate-limited through the shared
 registry, SSRF-guarded (scheme + private-address rejection), and size-capped.
 
 Configuration (environment):
 * ``OBSIGATE_SEARXNG_URL``   — defaults to https://search.dracodev.net
 * ``OBSIGATE_WEB_TIMEOUT``   — seconds, default 10
+* ``OBSIGATE_WEB_FALLBACK``  — ``0``/``false`` disables the keyless HTML
+  fallbacks (SearXNG only), default enabled
 """
 
 from __future__ import annotations
 
+import base64
+import binascii
 import html as html_lib
 import ipaddress
 import logging
 import os
 import re
 import socket
+from collections.abc import Callable
 from typing import Any
-from urllib.parse import urlparse
+from urllib.parse import parse_qs, urlparse
 
 import httpx
 
@@ -34,7 +42,18 @@ logger = logging.getLogger("obsigate.tools.web")
 
 SEARXNG_URL = os.environ.get("OBSIGATE_SEARXNG_URL", "https://search.dracodev.net")
 WEB_TIMEOUT = float(os.environ.get("OBSIGATE_WEB_TIMEOUT", "10"))
+WEB_FALLBACK_ENABLED = os.environ.get("OBSIGATE_WEB_FALLBACK", "1").strip().lower() not in {
+    "0",
+    "false",
+    "no",
+    "off",
+}
 USER_AGENT = "ObsiGateAssistant/1.0 (+self-hosted vault AI)"
+# Search engines reject non-browser agents on their public HTML endpoints.
+BROWSER_UA = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+)
 MAX_FETCH_BYTES = 1_500_000
 MAX_TEXT_CHARS = 20_000
 
@@ -46,6 +65,18 @@ _BLOCK_SPLIT_RE = re.compile(
     r"</?(?:p|div|br|li|h[1-6]|tr|table|ul|ol|section|article|header|footer)\b[^>]*>",
     re.IGNORECASE,
 )
+_DDG_RESULT_RE = re.compile(
+    r'<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>(.*?)</a>', re.IGNORECASE | re.DOTALL
+)
+_DDG_SNIPPET_RE = re.compile(
+    r'<a[^>]*class="result__snippet"[^>]*>(.*?)</a>', re.IGNORECASE | re.DOTALL
+)
+_BING_RESULT_RE = re.compile(
+    r'<h2[^>]*>\s*<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', re.IGNORECASE | re.DOTALL
+)
+_BING_SNIPPET_RE = re.compile(
+    r'<p class="b_lineclamp[^"]*">(.*?)</p>', re.IGNORECASE | re.DOTALL
+)
 
 
 class SSRFError(ToolError):
@@ -99,6 +130,160 @@ def _html_to_text(raw: str) -> str:
     return text.strip()
 
 
+def _response_text(resp: httpx.Response) -> str:
+    """Decode a response body without relying on ``resp.text`` (easier to mock)."""
+    return resp.content.decode(resp.encoding or "utf-8", errors="replace")
+
+
+def _clean_fragment(fragment: str) -> str:
+    return html_lib.unescape(_TAG_RE.sub("", fragment)).strip()
+
+
+def _result(
+    title: str, url: str, snippet: str, published: Any = None, score: Any = None
+) -> dict[str, Any]:
+    return {
+        "title": (title or "")[:300],
+        "url": url or "",
+        "snippet": (snippet or "")[:600],
+        "published": published,
+        "score": score,
+    }
+
+
+def _search_searxng(
+    query: str, params: WebSearchInput
+) -> tuple[list[dict[str, Any]], list[str]]:
+    """Query the self-hosted SearXNG instance (JSON API)."""
+    url = SEARXNG_URL.rstrip("/") + "/search"
+    resp = httpx.get(
+        url,
+        params={
+            "q": query,
+            "format": "json",
+            "categories": params.category or "general",
+            "pageno": max(1, params.page),
+            **({"language": params.language} if params.language else {}),
+            "safesearch": "1",
+        },
+        headers={"User-Agent": USER_AGENT},
+        timeout=WEB_TIMEOUT,
+        follow_redirects=False,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    results = [
+        _result(
+            item.get("title") or "",
+            item.get("url") or "",
+            item.get("content") or "",
+            item.get("publishedDate"),
+            item.get("score"),
+        )
+        for item in (data.get("results") or [])[: params.max_results]
+    ]
+    unresponsive = [
+        name for entry in (data.get("unresponsive_engines") or [])
+        for name in ([entry[0]] if isinstance(entry, (list, tuple)) and entry else [entry])
+        if isinstance(name, str)
+    ]
+    return results, unresponsive
+
+
+def _unwrap_duckduckgo_url(href: str) -> str:
+    """DuckDuckGo HTML wraps hits in ``/l/?uddg=<urlencoded target>``."""
+    href = html_lib.unescape(href)
+    if href.startswith("//"):
+        href = "https:" + href
+    if "uddg=" in href:
+        values = parse_qs(urlparse(href).query).get("uddg")
+        if values:
+            return values[0]
+    return href
+
+
+def _search_duckduckgo(
+    query: str, params: WebSearchInput
+) -> tuple[list[dict[str, Any]], list[str]]:
+    """Keyless fallback: scrape the DuckDuckGo no-JS HTML endpoint."""
+    resp = httpx.get(
+        "https://html.duckduckgo.com/html/",
+        params={"q": query, **({"kl": params.language} if params.language else {})},
+        headers={"User-Agent": BROWSER_UA, "Accept": "text/html,application/xhtml+xml"},
+        timeout=WEB_TIMEOUT,
+        follow_redirects=False,
+    )
+    resp.raise_for_status()
+    body = _response_text(resp)
+    snippets = [_clean_fragment(m.group(1)) for m in _DDG_SNIPPET_RE.finditer(body)]
+    results: list[dict[str, Any]] = []
+    for index, match in enumerate(_DDG_RESULT_RE.finditer(body)):
+        results.append(
+            _result(
+                _clean_fragment(match.group(2)),
+                _unwrap_duckduckgo_url(match.group(1)),
+                snippets[index] if index < len(snippets) else "",
+            )
+        )
+        if len(results) >= params.max_results:
+            break
+    return results, []
+
+
+def _unwrap_bing_url(href: str) -> str:
+    """Bing wraps hits in ``/ck/a?...&u=a1<base64url target>``."""
+    href = html_lib.unescape(href)
+    match = re.search(r"[?&]u=a1([A-Za-z0-9_\-]+)", href)
+    if not match:
+        return href
+    token = match.group(1).replace("-", "+").replace("_", "/")
+    token += "=" * (-len(token) % 4)
+    try:
+        return base64.b64decode(token).decode("utf-8", errors="replace")
+    except (ValueError, binascii.Error):
+        return href
+
+
+def _search_bing(
+    query: str, params: WebSearchInput
+) -> tuple[list[dict[str, Any]], list[str]]:
+    """Last-resort keyless fallback: scrape Bing's result page."""
+    resp = httpx.get(
+        "https://www.bing.com/search",
+        params={"q": query, **({"setlang": params.language} if params.language else {})},
+        headers={"User-Agent": BROWSER_UA, "Accept": "text/html,application/xhtml+xml"},
+        timeout=WEB_TIMEOUT,
+        follow_redirects=False,
+    )
+    resp.raise_for_status()
+    body = _response_text(resp)
+    snippets = [_clean_fragment(m.group(1)) for m in _BING_SNIPPET_RE.finditer(body)]
+    results: list[dict[str, Any]] = []
+    for index, match in enumerate(_BING_RESULT_RE.finditer(body)):
+        results.append(
+            _result(
+                _clean_fragment(match.group(2)),
+                _unwrap_bing_url(match.group(1)),
+                snippets[index] if index < len(snippets) else "",
+            )
+        )
+        if len(results) >= params.max_results:
+            break
+    return results, []
+
+
+_Provider = Callable[[str, WebSearchInput], "tuple[list[dict[str, Any]], list[str]]"]
+
+
+def _provider_chain() -> list[tuple[str, _Provider]]:
+    """Ordered providers: self-hosted meta-search first, then keyless fallbacks."""
+    chain: list[tuple[str, _Provider]] = [("searxng", _search_searxng)]
+    if WEB_FALLBACK_ENABLED:
+        chain.append(("duckduckgo", _search_duckduckgo))
+        chain.append(("bing", _search_bing))
+    return chain
+
+
 @tool(
     name="web_search",
     description=(
@@ -111,67 +296,59 @@ def _html_to_text(raw: str) -> str:
     scopes=(ToolScope.IN_APP,),
 )
 def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
-    """Query the self-hosted SearXNG instance and return trimmed results."""
+    """Try each configured provider and return the first non-empty result set."""
     query = params.query.strip()
     if not query:
         raise ToolError("Requête vide", code="invalid_arguments")
-    url = SEARXNG_URL.rstrip("/") + "/search"
-    try:
-        resp = httpx.get(
-            url,
-            params={
-                "q": query,
-                "format": "json",
-                "categories": params.category or "general",
-                "pageno": max(1, params.page),
-                **({"language": params.language} if params.language else {}),
-                "safesearch": "1",
-            },
-            headers={"User-Agent": USER_AGENT},
-            timeout=WEB_TIMEOUT,
-            follow_redirects=False,
-        )
-        resp.raise_for_status()
-        data = resp.json()
-    except httpx.HTTPError as e:
-        logger.warning("web_search failed: %s", e)
+
+    attempts: list[str] = []
+    unresponsive: list[str] = []
+    reachable = False
+    last_error: Exception | None = None
+
+    for name, provider in _provider_chain():
+        attempts.append(name)
+        try:
+            results, engines = provider(query, params)
+        except (httpx.HTTPError, ValueError, AttributeError) as e:
+            logger.warning("web_search provider %s failed: %s", name, e)
+            last_error = e
+            continue
+        reachable = True
+        if engines:
+            unresponsive = engines
+        if results:
+            payload: dict[str, Any] = {
+                "query": query,
+                "provider": name,
+                "results": results,
+                "count": len(results),
+            }
+            if unresponsive:
+                payload["unresponsive_engines"] = unresponsive[:8]
+            return payload
+
+    if not reachable:
         raise ToolError(
             "Le moteur de recherche web est momentanément indisponible.",
             code="web_search_unavailable",
-        ) from e
-    results: list[dict[str, Any]] = []
-    for item in (data.get("results") or [])[: params.max_results]:
-        results.append(
-            {
-                "title": (item.get("title") or "")[:300],
-                "url": item.get("url") or "",
-                "snippet": (item.get("content") or "")[:600],
-                "published": item.get("publishedDate"),
-                "score": item.get("score"),
-            }
-        )
-    unresponsive = [
-        name for entry in (data.get("unresponsive_engines") or [])
-        for name in ([entry[0]] if isinstance(entry, (list, tuple)) and entry else [entry])
-        if isinstance(name, str)
-    ]
-    payload: dict[str, Any] = {
+        ) from last_error
+
+    # Every provider answered but returned nothing: tell the model explicitly
+    # so it stops retrying the same query until its tool quota burns out.
+    payload = {
         "query": query,
-        "engine": "searxng",
-        "results": results,
-        "count": len(results),
+        "provider": attempts[-1],
+        "results": [],
+        "count": 0,
+        "warning": (
+            "Aucun résultat : les fournisseurs de recherche web sont "
+            f"indisponibles ({', '.join(attempts)}). "
+            "Ne relance pas la même recherche — dis-le à l'utilisateur."
+        ),
     }
     if unresponsive:
         payload["unresponsive_engines"] = unresponsive[:8]
-    if not results:
-        # An instance whose upstream engines are all blocked (CAPTCHA / rate
-        # limit) answers 200 with an empty list. Without an explicit hint the
-        # model retries the same search until it burns its tool quota.
-        payload["warning"] = (
-            "Aucun résultat : les moteurs de recherche de l'instance SearXNG sont "
-            f"indisponibles ({', '.join(unresponsive[:5]) or 'inconnus'}). "
-            "Ne relance pas la même recherche — dis-le à l'utilisateur."
-        )
     return payload