feat(assistant): #91 steps humains en direct + outils web_search/fetch_url (labels backend, SSE step, garde SSRF)
This commit is contained in:
@@ -0,0 +1,217 @@
|
||||
"""Web tools for the assistant (Notion-style "research" capabilities).
|
||||
|
||||
Phase 1 of the documented web-toolset roadmap:
|
||||
|
||||
* ``web_search`` — query the self-hosted SearXNG instance (no API key).
|
||||
* ``fetch_url`` — retrieve a public web page and return readable text.
|
||||
|
||||
Both are READ-risk tools (no confirmation), rate-limited through the shared
|
||||
registry, SSRF-guarded (scheme + private-address rejection), and size-capped.
|
||||
|
||||
Configuration (environment):
|
||||
* ``OBSIGATE_SEARXNG_URL`` — defaults to https://search.dracodev.net
|
||||
* ``OBSIGATE_WEB_TIMEOUT`` — seconds, default 10
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html as html_lib
|
||||
import ipaddress
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.tools.context import ToolError, ToolRisk, ToolScope
|
||||
from backend.tools.registry import tool
|
||||
from backend.tools.schemas import FetchUrlInput, WebSearchInput
|
||||
|
||||
logger = logging.getLogger("obsigate.tools.web")
|
||||
|
||||
SEARXNG_URL = os.environ.get("OBSIGATE_SEARXNG_URL", "https://search.dracodev.net")
|
||||
WEB_TIMEOUT = float(os.environ.get("OBSIGATE_WEB_TIMEOUT", "10"))
|
||||
USER_AGENT = "ObsiGateAssistant/1.0 (+self-hosted vault AI)"
|
||||
MAX_FETCH_BYTES = 1_500_000
|
||||
MAX_TEXT_CHARS = 20_000
|
||||
|
||||
_BLOCKED_TAGS_RE = re.compile(
|
||||
r"<(script|style|noscript|template|svg)\b.*?</\1>", re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
_TAG_RE = re.compile(r"<[^>]+>")
|
||||
_BLOCK_SPLIT_RE = re.compile(
|
||||
r"</?(?:p|div|br|li|h[1-6]|tr|table|ul|ol|section|article|header|footer)\b[^>]*>",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
class SSRFError(ToolError):
|
||||
"""Raised for a URL whose host is private/loopback or scheme unsupported."""
|
||||
|
||||
|
||||
def _assert_public_http_url(url: str) -> str:
|
||||
"""Reject non-http(s) schemes and private/loopback/link-local targets."""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except ValueError as e:
|
||||
raise SSRFError("URL invalide", code="invalid_url") from e
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
raise SSRFError("Seuls les schémas http/https sont autorisés", code="invalid_scheme")
|
||||
host = parsed.hostname
|
||||
if not host:
|
||||
raise SSRFError("URL sans hôte", code="invalid_url")
|
||||
# Resolve the host so DNS-rebinding to internal IPs is also caught.
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, None)
|
||||
except socket.gaierror as e:
|
||||
raise SSRFError(f"Hôte introuvable: {host}", code="dns_error") from e
|
||||
for info in infos:
|
||||
ip = ipaddress.ip_address(info[4][0])
|
||||
if (
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_reserved
|
||||
or ip.is_multicast
|
||||
or ip.is_unspecified
|
||||
):
|
||||
raise SSRFError("Accès aux adresses internes interdit", code="ssrf_blocked")
|
||||
return url
|
||||
|
||||
|
||||
def _html_to_text(raw: str) -> str:
|
||||
"""Cheap HTML → readable text: strip scripts/styles, tags, then compress.
|
||||
|
||||
Comments (which may carry script-like payloads) are removed first.
|
||||
"""
|
||||
text = re.sub(r"<!--.*?-->", " ", raw, flags=re.DOTALL)
|
||||
text = _BLOCKED_TAGS_RE.sub(" ", text)
|
||||
# Keep block boundaries as newlines before dropping the remaining tags.
|
||||
text = _BLOCK_SPLIT_RE.sub("\n", text)
|
||||
text = _TAG_RE.sub("", text)
|
||||
text = html_lib.unescape(text)
|
||||
text = re.sub(r"[ \t]+", " ", text)
|
||||
text = re.sub(r" ?\n ?", "\n", text)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
@tool(
|
||||
name="web_search",
|
||||
description=(
|
||||
"Search the public web for current information and return ranked results "
|
||||
"(title, url, snippet). Use for facts outside the vault: weather, news, "
|
||||
"documentation, versions, prices, anything that needs live sources."
|
||||
),
|
||||
input_model=WebSearchInput,
|
||||
risk=ToolRisk.READ,
|
||||
scopes=(ToolScope.IN_APP,),
|
||||
)
|
||||
def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
|
||||
"""Query the self-hosted SearXNG instance and return trimmed results."""
|
||||
query = params.query.strip()
|
||||
if not query:
|
||||
raise ToolError("Requête vide", code="invalid_arguments")
|
||||
url = SEARXNG_URL.rstrip("/") + "/search"
|
||||
try:
|
||||
resp = httpx.get(
|
||||
url,
|
||||
params={
|
||||
"q": query,
|
||||
"format": "json",
|
||||
"categories": params.category or "general",
|
||||
"pageno": max(1, params.page),
|
||||
**({"language": params.language} if params.language else {}),
|
||||
"safesearch": "1",
|
||||
},
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
timeout=WEB_TIMEOUT,
|
||||
follow_redirects=False,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except httpx.HTTPError as e:
|
||||
logger.warning("web_search failed: %s", e)
|
||||
raise ToolError(
|
||||
"Le moteur de recherche web est momentanément indisponible.",
|
||||
code="web_search_unavailable",
|
||||
) from e
|
||||
results: list[dict[str, Any]] = []
|
||||
for item in (data.get("results") or [])[: params.max_results]:
|
||||
results.append(
|
||||
{
|
||||
"title": (item.get("title") or "")[:300],
|
||||
"url": item.get("url") or "",
|
||||
"snippet": (item.get("content") or "")[:600],
|
||||
"published": item.get("publishedDate"),
|
||||
"score": item.get("score"),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"query": query,
|
||||
"engine": "searxng",
|
||||
"results": results,
|
||||
"count": len(results),
|
||||
}
|
||||
|
||||
|
||||
@tool(
|
||||
name="fetch_url",
|
||||
description=(
|
||||
"Fetch a public web page (http/https) and return its readable text. "
|
||||
"Use after web_search to read a promising result in detail. HTML is "
|
||||
"converted to plain text; binary pages are rejected."
|
||||
),
|
||||
input_model=FetchUrlInput,
|
||||
risk=ToolRisk.READ,
|
||||
scopes=(ToolScope.IN_APP,),
|
||||
)
|
||||
def fetch_url(ctx, params: FetchUrlInput) -> dict[str, Any]:
|
||||
"""Retrieve one page, guard against SSRF, and extract its text."""
|
||||
url = _assert_public_http_url(params.url.strip())
|
||||
try:
|
||||
# Follow redirects manually so every hop is re-checked against the
|
||||
# private-address SSRF guard (a public page can redirect to 127.0.0.1).
|
||||
resp = None
|
||||
for _hop in range(5):
|
||||
resp = httpx.get(
|
||||
url,
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "text/html,application/xhtml+xml,*/*"},
|
||||
timeout=WEB_TIMEOUT,
|
||||
follow_redirects=False,
|
||||
)
|
||||
if resp.status_code in (301, 302, 303, 307, 308):
|
||||
location = resp.headers.get("location") or ""
|
||||
if not location:
|
||||
break
|
||||
url = str(httpx.URL(url).join(location))
|
||||
url = _assert_public_http_url(url)
|
||||
continue
|
||||
break
|
||||
assert resp is not None
|
||||
resp.raise_for_status()
|
||||
except SSRFError:
|
||||
raise
|
||||
except httpx.HTTPError as e:
|
||||
logger.warning("fetch_url failed for %s: %s", url, e)
|
||||
raise ToolError("Impossible de récupérer la page.", code="fetch_unavailable") from e
|
||||
ctype = (resp.headers.get("content-type") or "").lower()
|
||||
if not any(t in ctype for t in ("html", "xml", "text", "json", "markdown")):
|
||||
raise ToolError(
|
||||
f"Type de contenu non pris en charge: {ctype.split(';')[0] or 'inconnu'}",
|
||||
code="unsupported_content_type",
|
||||
)
|
||||
raw = (resp.content[:MAX_FETCH_BYTES]).decode(resp.encoding or "utf-8", errors="replace")
|
||||
title_match = re.search(r"<title[^>]*>(.*?)</title>", raw, re.IGNORECASE | re.DOTALL)
|
||||
title = html_lib.unescape(title_match.group(1)).strip()[:300] if title_match else ""
|
||||
text = _html_to_text(raw)[:MAX_TEXT_CHARS]
|
||||
return {
|
||||
"url": str(resp.url),
|
||||
"status": resp.status_code,
|
||||
"title": title,
|
||||
"text": text,
|
||||
"truncated": len(raw) > MAX_TEXT_CHARS,
|
||||
}
|
||||
Reference in New Issue
Block a user