235 lines
8.6 KiB
Python
235 lines
8.6 KiB
Python
"""Web tools for the assistant (Notion-style "research" capabilities).
|
|
|
|
Phase 1 of the documented web-toolset roadmap:
|
|
|
|
* ``web_search`` — query the self-hosted SearXNG instance (no API key).
|
|
* ``fetch_url`` — retrieve a public web page and return readable text.
|
|
|
|
Both are READ-risk tools (no confirmation), rate-limited through the shared
|
|
registry, SSRF-guarded (scheme + private-address rejection), and size-capped.
|
|
|
|
Configuration (environment):
|
|
* ``OBSIGATE_SEARXNG_URL`` — defaults to https://search.dracodev.net
|
|
* ``OBSIGATE_WEB_TIMEOUT`` — seconds, default 10
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html as html_lib
|
|
import ipaddress
|
|
import logging
|
|
import os
|
|
import re
|
|
import socket
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
import httpx
|
|
|
|
from backend.tools.context import ToolError, ToolRisk, ToolScope
|
|
from backend.tools.registry import tool
|
|
from backend.tools.schemas import FetchUrlInput, WebSearchInput
|
|
|
|
logger = logging.getLogger("obsigate.tools.web")
|
|
|
|
SEARXNG_URL = os.environ.get("OBSIGATE_SEARXNG_URL", "https://search.dracodev.net")
|
|
WEB_TIMEOUT = float(os.environ.get("OBSIGATE_WEB_TIMEOUT", "10"))
|
|
USER_AGENT = "ObsiGateAssistant/1.0 (+self-hosted vault AI)"
|
|
MAX_FETCH_BYTES = 1_500_000
|
|
MAX_TEXT_CHARS = 20_000
|
|
|
|
_BLOCKED_TAGS_RE = re.compile(
|
|
r"<(script|style|noscript|template|svg)\b.*?</\1>", re.IGNORECASE | re.DOTALL
|
|
)
|
|
_TAG_RE = re.compile(r"<[^>]+>")
|
|
_BLOCK_SPLIT_RE = re.compile(
|
|
r"</?(?:p|div|br|li|h[1-6]|tr|table|ul|ol|section|article|header|footer)\b[^>]*>",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
class SSRFError(ToolError):
|
|
"""Raised for a URL whose host is private/loopback or scheme unsupported."""
|
|
|
|
|
|
def _assert_public_http_url(url: str) -> str:
|
|
"""Reject non-http(s) schemes and private/loopback/link-local targets."""
|
|
try:
|
|
parsed = urlparse(url)
|
|
except ValueError as e:
|
|
raise SSRFError("URL invalide", code="invalid_url") from e
|
|
if parsed.scheme not in ("http", "https"):
|
|
raise SSRFError("Seuls les schémas http/https sont autorisés", code="invalid_scheme")
|
|
host = parsed.hostname
|
|
if not host:
|
|
raise SSRFError("URL sans hôte", code="invalid_url")
|
|
# Resolve the host so DNS-rebinding to internal IPs is also caught.
|
|
try:
|
|
infos = socket.getaddrinfo(host, None)
|
|
except socket.gaierror as e:
|
|
raise SSRFError(f"Hôte introuvable: {host}", code="dns_error") from e
|
|
for info in infos:
|
|
ip = ipaddress.ip_address(info[4][0])
|
|
if (
|
|
ip.is_private
|
|
or ip.is_loopback
|
|
or ip.is_link_local
|
|
or ip.is_reserved
|
|
or ip.is_multicast
|
|
or ip.is_unspecified
|
|
):
|
|
raise SSRFError("Accès aux adresses internes interdit", code="ssrf_blocked")
|
|
return url
|
|
|
|
|
|
def _html_to_text(raw: str) -> str:
|
|
"""Cheap HTML → readable text: strip scripts/styles, tags, then compress.
|
|
|
|
Comments (which may carry script-like payloads) are removed first.
|
|
"""
|
|
text = re.sub(r"<!--.*?-->", " ", raw, flags=re.DOTALL)
|
|
text = _BLOCKED_TAGS_RE.sub(" ", text)
|
|
# Keep block boundaries as newlines before dropping the remaining tags.
|
|
text = _BLOCK_SPLIT_RE.sub("\n", text)
|
|
text = _TAG_RE.sub("", text)
|
|
text = html_lib.unescape(text)
|
|
text = re.sub(r"[ \t]+", " ", text)
|
|
text = re.sub(r" ?\n ?", "\n", text)
|
|
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
return text.strip()
|
|
|
|
|
|
@tool(
|
|
name="web_search",
|
|
description=(
|
|
"Search the public web for current information and return ranked results "
|
|
"(title, url, snippet). Use for facts outside the vault: weather, news, "
|
|
"documentation, versions, prices, anything that needs live sources."
|
|
),
|
|
input_model=WebSearchInput,
|
|
risk=ToolRisk.READ,
|
|
scopes=(ToolScope.IN_APP,),
|
|
)
|
|
def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
|
|
"""Query the self-hosted SearXNG instance and return trimmed results."""
|
|
query = params.query.strip()
|
|
if not query:
|
|
raise ToolError("Requête vide", code="invalid_arguments")
|
|
url = SEARXNG_URL.rstrip("/") + "/search"
|
|
try:
|
|
resp = httpx.get(
|
|
url,
|
|
params={
|
|
"q": query,
|
|
"format": "json",
|
|
"categories": params.category or "general",
|
|
"pageno": max(1, params.page),
|
|
**({"language": params.language} if params.language else {}),
|
|
"safesearch": "1",
|
|
},
|
|
headers={"User-Agent": USER_AGENT},
|
|
timeout=WEB_TIMEOUT,
|
|
follow_redirects=False,
|
|
)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
except httpx.HTTPError as e:
|
|
logger.warning("web_search failed: %s", e)
|
|
raise ToolError(
|
|
"Le moteur de recherche web est momentanément indisponible.",
|
|
code="web_search_unavailable",
|
|
) from e
|
|
results: list[dict[str, Any]] = []
|
|
for item in (data.get("results") or [])[: params.max_results]:
|
|
results.append(
|
|
{
|
|
"title": (item.get("title") or "")[:300],
|
|
"url": item.get("url") or "",
|
|
"snippet": (item.get("content") or "")[:600],
|
|
"published": item.get("publishedDate"),
|
|
"score": item.get("score"),
|
|
}
|
|
)
|
|
unresponsive = [
|
|
name for entry in (data.get("unresponsive_engines") or [])
|
|
for name in ([entry[0]] if isinstance(entry, (list, tuple)) and entry else [entry])
|
|
if isinstance(name, str)
|
|
]
|
|
payload: dict[str, Any] = {
|
|
"query": query,
|
|
"engine": "searxng",
|
|
"results": results,
|
|
"count": len(results),
|
|
}
|
|
if unresponsive:
|
|
payload["unresponsive_engines"] = unresponsive[:8]
|
|
if not results:
|
|
# An instance whose upstream engines are all blocked (CAPTCHA / rate
|
|
# limit) answers 200 with an empty list. Without an explicit hint the
|
|
# model retries the same search until it burns its tool quota.
|
|
payload["warning"] = (
|
|
"Aucun résultat : les moteurs de recherche de l'instance SearXNG sont "
|
|
f"indisponibles ({', '.join(unresponsive[:5]) or 'inconnus'}). "
|
|
"Ne relance pas la même recherche — dis-le à l'utilisateur."
|
|
)
|
|
return payload
|
|
|
|
|
|
@tool(
|
|
name="fetch_url",
|
|
description=(
|
|
"Fetch a public web page (http/https) and return its readable text. "
|
|
"Use after web_search to read a promising result in detail. HTML is "
|
|
"converted to plain text; binary pages are rejected."
|
|
),
|
|
input_model=FetchUrlInput,
|
|
risk=ToolRisk.READ,
|
|
scopes=(ToolScope.IN_APP,),
|
|
)
|
|
def fetch_url(ctx, params: FetchUrlInput) -> dict[str, Any]:
|
|
"""Retrieve one page, guard against SSRF, and extract its text."""
|
|
url = _assert_public_http_url(params.url.strip())
|
|
try:
|
|
# Follow redirects manually so every hop is re-checked against the
|
|
# private-address SSRF guard (a public page can redirect to 127.0.0.1).
|
|
resp = None
|
|
for _hop in range(5):
|
|
resp = httpx.get(
|
|
url,
|
|
headers={"User-Agent": USER_AGENT, "Accept": "text/html,application/xhtml+xml,*/*"},
|
|
timeout=WEB_TIMEOUT,
|
|
follow_redirects=False,
|
|
)
|
|
if resp.status_code in (301, 302, 303, 307, 308):
|
|
location = resp.headers.get("location") or ""
|
|
if not location:
|
|
break
|
|
url = str(httpx.URL(url).join(location))
|
|
url = _assert_public_http_url(url)
|
|
continue
|
|
break
|
|
assert resp is not None
|
|
resp.raise_for_status()
|
|
except SSRFError:
|
|
raise
|
|
except httpx.HTTPError as e:
|
|
logger.warning("fetch_url failed for %s: %s", url, e)
|
|
raise ToolError("Impossible de récupérer la page.", code="fetch_unavailable") from e
|
|
ctype = (resp.headers.get("content-type") or "").lower()
|
|
if not any(t in ctype for t in ("html", "xml", "text", "json", "markdown")):
|
|
raise ToolError(
|
|
f"Type de contenu non pris en charge: {ctype.split(';')[0] or 'inconnu'}",
|
|
code="unsupported_content_type",
|
|
)
|
|
raw = (resp.content[:MAX_FETCH_BYTES]).decode(resp.encoding or "utf-8", errors="replace")
|
|
title_match = re.search(r"<title[^>]*>(.*?)</title>", raw, re.IGNORECASE | re.DOTALL)
|
|
title = html_lib.unescape(title_match.group(1)).strip()[:300] if title_match else ""
|
|
text = _html_to_text(raw)[:MAX_TEXT_CHARS]
|
|
return {
|
|
"url": str(resp.url),
|
|
"status": resp.status_code,
|
|
"title": title,
|
|
"text": text,
|
|
"truncated": len(raw) > MAX_TEXT_CHARS,
|
|
}
|