101 lines
3.4 KiB
Python
101 lines
3.4 KiB
Python
"""Dynamic page rendering (Playwright) — ``fetch_url(render=True)``.
|
|
|
|
Static pages are fetched with httpx inside :mod:`backend.tools.web`. Dynamic
|
|
pages (SPA/React, JS-loaded content) need a real browser engine; this module
|
|
runs one Playwright call inside a dedicated worker thread so browser
|
|
crashes/timeouts never take over the tool layer, and the heavyweight
|
|
dependency stays optional:
|
|
|
|
* not installed → ``ToolError(code="playwright_unavailable")`` with a clear
|
|
message (the assistant explains the limitation instead of hanging);
|
|
* installed → ``pip install playwright && playwright install chromium``.
|
|
|
|
The SSRF guard (scheme + private-address rejection) is applied before the
|
|
browser navigates. Note: unlike the httpx path, internal redirects performed
|
|
by the browser engine are not re-checked hop by hop.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html as html_lib
|
|
import logging
|
|
import re
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import Any
|
|
|
|
from backend.tools.context import ToolError
|
|
from backend.tools.web import (
|
|
MAX_TEXT_CHARS,
|
|
USER_AGENT,
|
|
_assert_public_http_url,
|
|
_html_to_text,
|
|
)
|
|
|
|
logger = logging.getLogger("obsigate.tools.webrender")
|
|
|
|
# One worker: browser automation is serialized on purpose (one Chromium at a
|
|
# time keeps memory predictable on small hosts).
|
|
_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="obsigate-playwright")
|
|
GOTO_TIMEOUT_MS = 20_000
|
|
|
|
|
|
def _playwright_available() -> bool:
|
|
try:
|
|
import playwright # noqa: F401
|
|
except ImportError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _render_in_worker(url: str) -> dict[str, Any]:
|
|
"""Synchronous Playwright render — runs in the dedicated worker thread."""
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
status = 0
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
try:
|
|
page = browser.new_page(user_agent=USER_AGENT)
|
|
response = page.goto(url, wait_until="networkidle", timeout=GOTO_TIMEOUT_MS)
|
|
if response is not None:
|
|
status = response.status
|
|
raw = page.content()
|
|
title = html_lib.unescape(page.title() or "").strip()
|
|
text = _html_to_text(raw)[:MAX_TEXT_CHARS]
|
|
finally:
|
|
browser.close()
|
|
title = re.sub(r"\s+", " ", title)[:300]
|
|
return {
|
|
"url": url,
|
|
"status": status,
|
|
"title": title,
|
|
"text": text,
|
|
"rendered": True,
|
|
"truncated": len(raw) > MAX_TEXT_CHARS,
|
|
}
|
|
|
|
|
|
def render_page(url: str) -> dict[str, Any]:
|
|
"""Render *url* (JavaScript included) and return readable text.
|
|
|
|
Raises:
|
|
ToolError: ``playwright_unavailable`` when the optional dependency is
|
|
missing, ``render_unavailable`` when the render itself failed.
|
|
"""
|
|
_assert_public_http_url(url)
|
|
if not _playwright_available():
|
|
raise ToolError(
|
|
"Rendu dynamique indisponible : Playwright n'est pas installé "
|
|
"(pip install playwright && playwright install chromium).",
|
|
code="playwright_unavailable",
|
|
)
|
|
try:
|
|
return _executor.submit(_render_in_worker, url).result(timeout=GOTO_TIMEOUT_MS / 1000 + 40)
|
|
except ToolError:
|
|
raise
|
|
except Exception as e:
|
|
logger.warning("render_page failed for %s: %s", url, e)
|
|
raise ToolError(
|
|
"Le rendu dynamique de la page a échoué.", code="render_unavailable"
|
|
) from e
|