"""FlowDeck — Web search + GitHub search pour les tools agent (v7.46.0). Alimente les deux tools réseau qui manquaient au registre : la recherche **en ligne** (`web_search`) et la recherche **GitHub** (`search_code`). Choix de conception : * **Provider configurable** (``settings.web_search_provider``) : Exa (recherche sémantique + snippets, clé requise) en priorité, DuckDuckGo en repli sans compte. Le repli parse du HTML — c'est volontairement *dégradé* : il n'est jamais utilisé si une clé Exa est configurée, et sert surtout à garder le skill utilisable sur une instance sans compte externe. * **Aucune dépendance ajoutée** : httpx déjà présent, ``shared_client`` pour le pool de connexions (A42), et les garde-fous SSRF déjà éprouvés par ``app.services.importers.url_fetch`` / ``og_fetcher``. * **Toujours des sources** : chaque résultat porte titre + URL + extrait, pour que l'IA puisse citer au lieu d'inventer. La couche réseau est injectable (``transport``) pour que les tests n'aient jamais besoin du réseau. """ from __future__ import annotations import html as htmlmod import logging import re from typing import Any from urllib.parse import parse_qs, urlparse from app.config import settings from app.services.http_client import shared_client logger = logging.getLogger(__name__) USER_AGENT = "FlowDeck-Agent/1.0 (+web_search)" #: Domaines de repli interdits : réponses « sous conditions » / anti-bot qui #: pollueraient les résultats (DuckDuckGo sert parfois une page de challenge). _BLOCKED_RESULT_HOSTS = frozenset({ "duckduckgo.com", "lite.duckduckgo.com", "duckduckgo.com.yahoo.net", }) EXA_ENDPOINT = "https://api.exa.ai/search" DDG_ENDPOINT = "https://lite.duckduckgo.com/lite/" GITHUB_API = "https://api.github.com" MAX_RESULTS = 10 _SNIPPET_CHARS = 300 def _clean(text: str | None) -> str: return re.sub(r"\s+", " ", (text or "")).strip() def _clip(text: str, limit: int = _SNIPPET_CHARS) -> str: text = _clean(text) return text if len(text) <= limit else text[: limit - 1].rstrip() + "…" def _is_resultable(url: str) -> bool: """Ignore les redirections vers le moteur lui-même et les URLs non http.""" parsed = urlparse(url or "") if parsed.scheme not in ("http", "https") or not parsed.hostname: return False return parsed.hostname.lower() not in _BLOCKED_RESULT_HOSTS # ══════════════════════ Exa ══════════════════════ async def _search_exa(query: str, num_results: int, transport=None) -> list[dict]: key = (settings.exa_api_key or "").strip() if not key: return [] payload = { "query": query, "numResults": max(1, min(num_results, MAX_RESULTS)), "contents": {"text": {"maxCharacters": 1200}}, } kwargs: dict[str, Any] = {"timeout": 15.0} if transport is not None: kwargs["transport"] = transport async with shared_client(**kwargs) as client: resp = await client.post( EXA_ENDPOINT, json=payload, headers={"x-api-key": key, "Content-Type": "application/json", "User-Agent": USER_AGENT}, ) if resp.status_code != 200: raise RuntimeError(f"Exa HTTP {resp.status_code}") results = [] for item in (resp.json() or {}).get("results", []): url = item.get("url") or "" if not _is_resultable(url): continue results.append({ "title": _clip(item.get("title") or url, 160), "url": url, "snippet": _clip(item.get("text") or item.get("summary") or ""), "published": (item.get("publishedDate") or "")[:10], "source": "exa", }) return results[:MAX_RESULTS] # ══════════════════════ DuckDuckGo (repli sans compte) ══════════════════════ _DDG_ROW_RE = re.compile( r"]+class=\"[^\"]*result-link[^\"]*\"[^>]+href=\"(?P[^\"]+)\"[^>]*>(?P.*?)</a>", re.I | re.S, ) _DDG_SNIPPET_RE = re.compile( r"class=\"[^\"]*result-snippet[^\"]*\"[^>]*>(?P<snippet>.*?)</td>", re.I | re.S ) _TAG_RE = re.compile(r"<[^>]+>") def _unwrap_ddg(href: str) -> str: """DuckDuckGo emballe les résultats dans ``/l/?uddg=<url encodé>``.""" if not href: return "" parsed = urlparse(htmlmod.unescape(href)) if "uddg" in (parsed.query or ""): values = parse_qs(parsed.query).get("uddg") if values: return values[0] return href async def _search_duckduckgo(query: str, num_results: int, transport=None) -> list[dict]: kwargs: dict[str, Any] = {"timeout": 12.0} if transport is not None: kwargs["transport"] = transport async with shared_client(**kwargs) as client: resp = await client.post( DDG_ENDPOINT, data={"q": query, "kl": "wt-wt"}, headers={"User-Agent": USER_AGENT, "Content-Type": "application/x-www-form-urlencoded"}, ) if resp.status_code != 200: raise RuntimeError(f"DuckDuckGo HTTP {resp.status_code}") body = resp.text links = list(_DDG_ROW_RE.finditer(body)) snippets = [_clean(_TAG_RE.sub("", m.group("snippet"))) for m in _DDG_SNIPPET_RE.finditer(body)] results = [] for idx, match in enumerate(links): url = _unwrap_ddg(match.group("href")) if not _is_resultable(url): continue results.append({ "title": _clean(_TAG_RE.sub("", match.group("title"))), "url": url, "snippet": _clip(snippets[idx]) if idx < len(snippets) else "", "published": "", "source": "duckduckgo", }) if len(results) >= min(num_results, MAX_RESULTS): break return results # ══════════════════════ API publique ══════════════════════ async def search_web(query: str, num_results: int = 5, transport=None) -> tuple[list[dict], str]: """Recherche web → ``(results, provider)``. Essaie le provider configuré puis l'autre ; si aucun ne répond, renvoie une liste vide — c'est au tool de traduire ça en message actionnable pour le modèle (« configurez EXA_API_KEY ») plutôt qu'en erreur réseau opaque. """ q = (query or "").strip() if not q: return [], "" provider = (settings.web_search_provider or "exa").strip().lower() order = ("exa", "duckduckgo") if provider == "exa" else ("duckduckgo", "exa") errors: list[str] = [] for name in order: try: if name == "exa": results = await _search_exa(q, num_results, transport) else: results = await _search_duckduckgo(q, num_results, transport) except Exception as exc: # noqa: BLE001 — un provider qui tombe ne doit pas tuer l'autre logger.info("web_search provider %s failed: %s", name, exc) errors.append(f"{name}: {exc}") continue if results: return results, name if errors: logger.warning("web_search sans résultat — %s", "; ".join(errors)) return [], order[0] return [], order[0] def available_providers() -> dict[str, bool]: """Quel provider est réellement utilisable (affiché dans les settings).""" return { "exa": bool((settings.exa_api_key or "").strip()), "duckduckgo": True, "github": True, # sans PAT : 10 req/min suffisent pour un usage agent } # ══════════════════════ GitHub ══════════════════════ _GH_KINDS = { "repositories": "repos", "repos": "repos", "code": "code", "issues": "issues", } def _gh_headers() -> dict[str, str]: headers = {"Accept": "application/vnd.github+json", "User-Agent": USER_AGENT, "X-GitHub-Api-Version": "2022-11-28"} token = (settings.github_token or "").strip() if token: headers["Authorization"] = f"Bearer {token}" return headers def _gh_item_name(item: dict, kind: str) -> str: if kind == "repos": return item.get("full_name") or item.get("name") or "" if kind == "code": return item.get("path") or "" return item.get("title") or f"#{item.get('number')}" def _gh_item_url(item: dict, kind: str) -> str: if kind == "code": # L'API code search ne renvoie pas d'URL exploitable : on reconstruit. repo = (item.get("repository") or {}).get("full_name", "") return f"https://github.com/{repo}/blob/HEAD/{item.get('path', '')}" if repo else "" return item.get("html_url") or "" def _gh_item_snippet(item: dict, kind: str) -> str: if kind == "code": return "" if kind == "repos": return _clip(item.get("description") or "") body = _clean(item.get("body") or "") return _clip(body) if body else _clip(item.get("title") or "") async def search_github(query: str, kind: str = "repositories", limit: int = 5, transport=None) -> list[dict]: """Recherche GitHub (repos | code | issues) → liste de résultats sourcés. Fonctionne sans PAT (limite 10 req/min) ; avec ``GITHUB_TOKEN``, 30 req/min. """ q = (query or "").strip() if not q: return [] k = _GH_KINDS.get((kind or "").strip().lower(), "repos") per_page = max(1, min(int(limit or 5), MAX_RESULTS)) kwargs: dict[str, Any] = {"timeout": 15.0} if transport is not None: kwargs["transport"] = transport async with shared_client(**kwargs) as client: resp = await client.get( f"{GITHUB_API}/search/{k}", params={"q": q, "per_page": per_page}, headers=_gh_headers(), ) if resp.status_code == 403: raise RuntimeError("GitHub: quota dépassé (403) — configurez GITHUB_TOKEN") if resp.status_code == 422: raise RuntimeError("GitHub: requête de recherche invalide (422)") if resp.status_code != 200: raise RuntimeError(f"GitHub HTTP {resp.status_code}") items = (resp.json() or {}).get("items", []) results = [] for item in items[:per_page]: results.append({ "title": _gh_item_name(item, k), "url": _gh_item_url(item, k), "snippet": _gh_item_snippet(item, k), "stars": item.get("stargazers_count") if k == "repos" else None, "state": item.get("state") if k == "issues" else "", "source": f"github/{k}", }) return results