"""Dynamic page rendering (Playwright) — ``fetch_url(render=True)``. Static pages are fetched with httpx inside :mod:`backend.tools.web`. Dynamic pages (SPA/React, JS-loaded content) need a real browser engine; this module runs one Playwright call inside a dedicated worker thread so browser crashes/timeouts never take over the tool layer, and the heavyweight dependency stays optional: * not installed → ``ToolError(code="playwright_unavailable")`` with a clear message (the assistant explains the limitation instead of hanging); * installed → ``pip install playwright && playwright install chromium``. The SSRF guard (scheme + private-address rejection) is applied before the browser navigates. Note: unlike the httpx path, internal redirects performed by the browser engine are not re-checked hop by hop. """ from __future__ import annotations import html as html_lib import logging import re from concurrent.futures import ThreadPoolExecutor from typing import Any from backend.tools.context import ToolError from backend.tools.web import ( MAX_TEXT_CHARS, USER_AGENT, _assert_public_http_url, _html_to_text, ) logger = logging.getLogger("obsigate.tools.webrender") # One worker: browser automation is serialized on purpose (one Chromium at a # time keeps memory predictable on small hosts). _executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="obsigate-playwright") GOTO_TIMEOUT_MS = 20_000 def _playwright_available() -> bool: try: import playwright # noqa: F401 except ImportError: return False return True def _render_in_worker(url: str) -> dict[str, Any]: """Synchronous Playwright render — runs in the dedicated worker thread.""" from playwright.sync_api import sync_playwright status = 0 with sync_playwright() as p: browser = p.chromium.launch(headless=True) try: page = browser.new_page(user_agent=USER_AGENT) response = page.goto(url, wait_until="networkidle", timeout=GOTO_TIMEOUT_MS) if response is not None: status = response.status raw = page.content() title = html_lib.unescape(page.title() or "").strip() text = _html_to_text(raw)[:MAX_TEXT_CHARS] finally: browser.close() title = re.sub(r"\s+", " ", title)[:300] return { "url": url, "status": status, "title": title, "text": text, "rendered": True, "truncated": len(raw) > MAX_TEXT_CHARS, } def render_page(url: str) -> dict[str, Any]: """Render *url* (JavaScript included) and return readable text. Raises: ToolError: ``playwright_unavailable`` when the optional dependency is missing, ``render_unavailable`` when the render itself failed. """ _assert_public_http_url(url) if not _playwright_available(): raise ToolError( "Rendu dynamique indisponible : Playwright n'est pas installé " "(pip install playwright && playwright install chromium).", code="playwright_unavailable", ) try: return _executor.submit(_render_in_worker, url).result(timeout=GOTO_TIMEOUT_MS / 1000 + 40) except ToolError: raise except Exception as e: logger.warning("render_page failed for %s: %s", url, e) raise ToolError( "Le rendu dynamique de la page a échoué.", code="render_unavailable" ) from e