feat(ai): ecosysteme d'outils phase 2 - recherche a cle, cache/retry, Playwright, crawl, Gitea/GitHub, documents (#92)

backend/tools/api.py

@@ -9,6 +9,9 @@ Note: ObsiGate uses implicit namespace packages (no tracked ``__init__.py``,
 which ``.gitignore`` excludes via ``_*.py``), hence this explicit facade.
 """
 
+from backend.tools import connected as _connected  # noqa: F401  (registers connected-source tools)
+from backend.tools import crawler as _crawler  # noqa: F401  (registers the site crawler)
+from backend.tools import documents as _documents  # noqa: F401  (registers document tools)
 from backend.tools import service as _service  # noqa: F401  (registers tools)
 from backend.tools import web as _web  # noqa: F401  (registers web tools)
 from backend.tools.context import (

backend/tools/connected.py

@@ -0,0 +1,239 @@
+"""Connected sources — Gitea & GitHub repositories (phase 2 #92).
+
+The assistant can query the source-hosting platforms the project actually
+uses (ObsiGate is hosted on Gitea): repositories, issues/pull requests and
+repository files. Everything is READ-risk, rate-limited through the shared
+registry and audited.
+
+Configuration (environment — injected by Infisical in production, never
+hard-coded):
+
+* ``OBSIGATE_GITEA_URL``    — base URL of the self-hosted instance (e.g.
+  ``https://git.example.net``); the ``gitea`` provider is only available when
+  this variable is set. Admin-controlled, so the SSRF guard does not apply
+  (unlike user-supplied URLs).
+* ``OBSIGATE_GITEA_TOKEN``  — optional personal access token (private repos).
+* ``OBSIGATE_GITHUB_TOKEN`` — optional token (raises the API rate limits and
+  unlocks private repositories).
+
+Cloud drives (Google Drive / OneDrive) deliberately stay out of the core:
+per the documented roadmap they are best served by an *external MCP server*
+(#79) so the OAuth surface remains outside ObsiGate.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import logging
+import os
+from typing import Any
+
+import httpx
+
+from backend.tools.context import ToolError, ToolRisk
+from backend.tools.registry import tool
+from backend.tools.schemas import GitGetFileInput, GitProviderInput, GitSearchIssuesInput
+
+logger = logging.getLogger("obsigate.tools.connected")
+
+TIMEOUT = 10.0
+USER_AGENT = "ObsiGateAssistant/1.0 (+self-hosted vault AI)"
+MAX_FILE_BYTES = 300_000
+GITHUB_API = "https://api.github.com"
+
+
+def _provider_base(provider: str) -> tuple[str, str]:
+    """Return (base_url, auth_header_value) for the requested provider."""
+    if provider == "gitea":
+        base = os.environ.get("OBSIGATE_GITEA_URL", "").strip().rstrip("/")
+        if not base:
+            raise ToolError(
+                "Source Gitea non configurée (OBSIGATE_GITEA_URL absente).",
+                code="provider_not_configured",
+            )
+        token = os.environ.get("OBSIGATE_GITEA_TOKEN", "").strip()
+        return base, f"token {token}" if token else ""
+    if provider == "github":
+        token = os.environ.get("OBSIGATE_GITHUB_TOKEN", "").strip()
+        return GITHUB_API, f"Bearer {token}" if token else ""
+    raise ToolError(
+        f"Fournisseur inconnu : {provider} ('gitea' ou 'github')",
+        code="invalid_arguments",
+    )
+
+
+def _headers(auth: str) -> dict[str, str]:
+    headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
+    if auth:
+        headers["Authorization"] = auth
+    return headers
+
+
+def _request(method: str, url: str, auth: str, **kwargs: Any) -> httpx.Response:
+    try:
+        resp = httpx.request(
+            method, url, headers=_headers(auth), timeout=TIMEOUT, follow_redirects=False,
+            **kwargs,
+        )
+    except httpx.HTTPError as e:
+        logger.warning("connected source request failed %s: %s", url, e)
+        raise ToolError(
+            "Source connectée momentanément indisponible.",
+            code="connected_source_unavailable",
+        ) from e
+    if resp.status_code in (401, 403):
+        raise ToolError(
+            "Accès refusé par la source connectée (jeton manquant ou expiré).",
+            code="permission_denied",
+        )
+    if resp.status_code == 404:
+        raise ToolError("Ressource introuvable sur la source connectée.", code="not_found")
+    resp.raise_for_status()
+    return resp
+
+
+def _normalize_repo(item: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "name": item.get("name") or "",
+        "full_name": item.get("full_name") or "",
+        "url": item.get("html_url") or item.get("clone_url") or "",
+        "description": item.get("description") or "",
+        "updated": item.get("updated_at") or "",
+        "private": bool(item.get("private", False)),
+    }
+
+
+@tool(
+    name="git_list_repos",
+    description=(
+        "List repositories on the connected Gitea instance or GitHub account "
+        "(name, url, description, last update). Use when the user asks about "
+        "their code projects."
+    ),
+    input_model=GitProviderInput,
+    risk=ToolRisk.READ,
+)
+def git_list_repos(ctx, params: GitProviderInput) -> dict[str, Any]:
+    """Query the configured source and return normalized repositories."""
+    base, auth = _provider_base(params.provider)
+    if params.provider == "gitea":
+        url = base + "/api/v1/repos/search"
+        query: dict[str, Any] = {"limit": params.limit}
+        if params.repo:
+            query["q"] = params.repo
+        resp = _request("GET", url, auth, params=query)
+        items = resp.json().get("data") or []
+    else:
+        if params.repo:
+            url = GITHUB_API + f"/repos/{params.repo.strip('/')}"
+            items = [_request("GET", url, auth).json()]
+        else:
+            resp = _request(
+                "GET", GITHUB_API + "/user/repos",
+                auth, params={"per_page": params.limit, "sort": "updated"},
+            )
+            items = resp.json()
+    repos = [_normalize_repo(item) for item in items if isinstance(item, dict)]
+    return {"provider": params.provider, "count": len(repos), "repos": repos}
+
+
+@tool(
+    name="git_search_issues",
+    description=(
+        "Search issues and pull requests on the connected Gitea instance or "
+        "GitHub (title/body keywords, optional repository scope, open/closed)."
+    ),
+    input_model=GitSearchIssuesInput,
+    risk=ToolRisk.READ,
+)
+def git_search_issues(ctx, params: GitSearchIssuesInput) -> dict[str, Any]:
+    """Query issues (and PRs) from the configured source."""
+    base, auth = _provider_base(params.provider)
+    state = params.state if params.state in ("open", "closed") else "open"
+    if params.provider == "gitea":
+        if params.repo:
+            url = base + f"/api/v1/repos/{params.repo.strip('/')}/issues"
+            query: dict[str, Any] = {"state": state, "limit": params.limit, "q": params.query}
+            resp = _request("GET", url, auth, params=query)
+            items = resp.json()
+        else:
+            url = base + "/api/v1/repos/issues/search"
+            resp = _request("GET", url, auth, params={
+                "q": params.query, "state": state, "limit": params.limit,
+            })
+            items = resp.json()
+    else:
+        clause = f"{params.query} is:issue is:{state}"
+        if params.repo:
+            clause += f" repo:{params.repo.strip('/')}"
+        resp = _request(
+            "GET", GITHUB_API + "/search/issues", auth,
+            params={"q": clause, "per_page": params.limit},
+        )
+        items = (resp.json().get("items") or [])
+    issues = [
+        {
+            "id": item.get("number") or item.get("id") or "",
+            "title": (item.get("title") or "")[:300],
+            "url": item.get("html_url") or "",
+            "state": item.get("state") or "",
+            "pull_request": bool(item.get("pull_request")),
+        }
+        for item in (items if isinstance(items, list) else [])
+        if isinstance(item, dict)
+    ]
+    return {
+        "provider": params.provider,
+        "query": params.query,
+        "count": len(issues),
+        "issues": issues,
+    }
+
+
+@tool(
+    name="git_get_file",
+    description=(
+        "Read a file's content from a connected Gitea or GitHub repository "
+        "(source code, docs, config). Text/JSON only, size-capped."
+    ),
+    input_model=GitGetFileInput,
+    risk=ToolRisk.READ,
+)
+def git_get_file(ctx, params: GitGetFileInput) -> dict[str, Any]:
+    """Fetch one repository file and return its decoded text content."""
+    base, auth = _provider_base(params.provider)
+    repo = params.repo.strip("/")
+    path = params.path.strip("/")
+    if not repo or not path:
+        raise ToolError(
+            "'repo' (owner/nom) et 'path' sont obligatoires", code="invalid_arguments"
+        )
+    if params.provider == "gitea":
+        url = base + f"/api/v1/repos/{repo}/contents/{path}"
+    else:
+        url = GITHUB_API + f"/repos/{repo}/contents/{path}"
+    if params.ref:
+        url += f"?ref={params.ref}"
+    resp = _request("GET", url, auth)
+    data = resp.json()
+    encoded = data.get("content") or ""
+    if (data.get("encoding") or "") == "base64" and encoded:
+        try:
+            content = base64.b64decode(encoded).decode("utf-8", errors="replace")
+        except (ValueError, binascii.Error) as e:
+            raise ToolError(
+                "Contenu du fichier illisible (encodage inattendu).",
+                code="file_decode_error",
+            ) from e
+    else:
+        content = encoded
+    truncated = len(content) > MAX_FILE_BYTES
+    return {
+        "provider": params.provider,
+        "repo": repo,
+        "path": data.get("path") or path,
+        "size": data.get("size") or len(content),
+        "content": content[:MAX_FILE_BYTES],
+        "truncated": truncated,
+    }

backend/tools/crawler.py

@@ -0,0 +1,197 @@
+"""Multi-page site crawl — ``crawl_site`` (phase 2 #92, WRITE + confirmation).
+
+The assistant can digest a small public site (documentation, docs portal) and
+store a Markdown summary inside a vault: one section per page, title, URL and
+readable text. The crawl is bounded and same-host only:
+
+* max 20 pages (``max_pages``), same hostname, breadth-first from the entry URL;
+* SSRF guard on every URL (scheme + private-address rejection), size caps;
+* no third-party crawler dependency (scrapy deliberately avoided — a bounded
+  httpx BFS keeps the surface small and the runtime predictable; the task is
+  executed as a single background-style tool run instead of a web request
+  pipeline).
+
+Risk is WRITE: the digest is written into a vault, so the two-step
+confirmation applies (Apply card in the UI, propose/apply over MCP).
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import httpx
+
+from backend.tools.context import ToolContext, ToolError, ToolRisk
+from backend.tools.registry import tool
+from backend.tools.schemas import CrawlSiteInput
+from backend.tools.web import (
+    USER_AGENT,
+    _assert_public_http_url,
+    _html_to_text,
+    _response_text,
+)
+
+logger = logging.getLogger("obsigate.tools.crawler")
+
+MAX_PAGE_BYTES = 800_000
+MAX_TOTAL_BYTES = 6_000_000
+MAX_TEXT_PER_PAGE = 12_000
+PAGE_TIMEOUT = 10.0
+_LINK_RE = re.compile(r'<a[^>]*href="([^"#]+)"', re.IGNORECASE)
+_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
+
+
+def _same_host(url: str, host: str) -> bool:
+    return (urlparse(url).hostname or "") == host
+
+
+def _extract_links(raw: str, base_url: str) -> list[str]:
+    import html as html_lib
+
+    links: list[str] = []
+    for match in _LINK_RE.finditer(raw):
+        href = html_lib.unescape(match.group(1)).strip()
+        if not href or href.lower().startswith(("javascript:", "mailto:", "tel:")):
+            continue
+        absolute = urljoin(base_url, href)
+        if absolute.lower().endswith((".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".pdf", ".zip")):
+            continue
+        links.append(absolute.split("#", 1)[0])
+    return links
+
+
+def _fetch_page(url: str) -> tuple[str, str]:
+    """Fetch one page (SSRF-guarded, manual redirects) → (title, text)."""
+    current = _assert_public_http_url(url)
+    resp = None
+    for _hop in range(5):
+        resp = httpx.get(
+            current,
+            headers={"User-Agent": USER_AGENT, "Accept": "text/html,*/*"},
+            timeout=PAGE_TIMEOUT,
+            follow_redirects=False,
+        )
+        if resp.status_code in (301, 302, 303, 307, 308):
+            location = resp.headers.get("location") or ""
+            if not location:
+                break
+            current = _assert_public_http_url(str(httpx.URL(current).join(location)))
+            continue
+        break
+    assert resp is not None
+    resp.raise_for_status()
+    ctype = (resp.headers.get("content-type") or "").lower()
+    if "html" not in ctype and "text" not in ctype:
+        raise ToolError(
+            f"Type de contenu non pris en charge: {ctype.split(';')[0] or 'inconnu'}",
+            code="unsupported_content_type",
+        )
+    raw = (resp.content[:MAX_PAGE_BYTES]).decode(resp.encoding or "utf-8", errors="replace")
+    title_match = _TITLE_RE.search(raw)
+    import html as html_lib
+
+    title = html_lib.unescape(title_match.group(1)).strip()[:300] if title_match else ""
+    return title, _html_to_text(raw)[:MAX_TEXT_PER_PAGE]
+
+
+@tool(
+    name="crawl_site",
+    description=(
+        "Crawl a small public site (same-host only, max 20 pages) starting at "
+        "a URL and save a Markdown digest (title, url, readable text per page) "
+        "into a vault. Use to capture an online documentation for offline use."
+    ),
+    input_model=CrawlSiteInput,
+    risk=ToolRisk.WRITE,
+    requires_vault=True,
+)
+def crawl_site(ctx: ToolContext, params: CrawlSiteInput) -> dict[str, Any]:
+    """Bounded BFS crawl; writes the digest file and returns a summary."""
+    from backend.services.errors import ServiceError
+    from backend.services.mutations import save_raw_file
+
+    start = _assert_public_http_url(params.url.strip())
+    host = urlparse(start).hostname or ""
+    if not host:
+        raise ToolError("URL sans hôte", code="invalid_url")
+
+    queue: list[str] = [start]
+    seen: set[str] = {start}
+    pages: list[dict[str, Any]] = []
+    total_bytes = 0
+    failures: list[str] = []
+
+    while queue and len(pages) < params.max_pages and total_bytes < MAX_TOTAL_BYTES:
+        url = queue.pop(0)
+        try:
+            title, text = _fetch_page(url)
+        except ToolError as e:
+            failures.append(url)
+            logger.warning("crawl_site page failed %s: %s", url, e.code)
+            continue
+        except httpx.HTTPError as e:
+            failures.append(url)
+            logger.warning("crawl_site page failed %s: %s", url, e)
+            continue
+        pages.append({"url": url, "title": title, "text": text})
+        total_bytes += len(text)
+        if len(pages) >= params.max_pages:
+            break
+        try:
+            raw_resp = httpx.get(
+                url, headers={"User-Agent": USER_AGENT}, timeout=PAGE_TIMEOUT,
+                follow_redirects=False,
+            )
+            raw = _response_text(raw_resp)
+        except (httpx.HTTPError, ValueError):
+            continue
+        for link in _extract_links(raw, url):
+            if len(pages) + len(queue) >= params.max_pages:
+                break
+            if link in seen or not _same_host(link, host):
+                continue
+            try:
+                _assert_public_http_url(link)
+            except ToolError:
+                continue
+            seen.add(link)
+            queue.append(link)
+
+    if not pages:
+        raise ToolError(
+            "Aucune page n'a pu être récupérée pour ce site.",
+            code="crawl_failed",
+        )
+
+    lines = [
+        f"# Crawl de {host}",
+        "",
+        f"> {len(pages)} page(s) capturée(s) depuis {start} — {time.strftime('%Y-%m-%d %H:%M')}",
+        "",
+    ]
+    for page in pages:
+        lines.append(f"## {page['title'] or page['url']}")
+        lines.append("")
+        lines.append(f"Source : {page['url']}")
+        lines.append("")
+        lines.append(page["text"])
+        lines.append("")
+    digest = "\n".join(lines).encode("utf-8")
+    try:
+        saved = save_raw_file(
+            params.vault, params.path, digest, overwrite=True, allow_docs=False
+        )
+    except ServiceError as e:
+        raise ToolError(e.message, code=e.code, details=e.details) from e
+    return {
+        "url": start,
+        "vault": params.vault,
+        "path": saved.get("path", params.path),
+        "pages": len(pages),
+        "failed": failures[:10],
+        "size": saved.get("size", len(digest)),
+    }

backend/tools/documents.py

@@ -0,0 +1,188 @@
+"""Document-production tools (phase 2 #92) — WRITE, confirmation required.
+
+The assistant can generate real files inside a vault:
+
+* ``create_xlsx`` — spreadsheet (openpyxl);
+* ``create_docx`` — Word document (python-docx);
+* ``create_csv``  — CSV (stdlib);
+* ``create_pdf``  — PDF (reportlab, from markdown-ish content).
+
+Every tool is ``WRITE`` (two-step confirm in the UI / propose-apply over MCP),
+vault-scoped through ``requires_vault`` and saved via the shared mutation
+service (path safety, read-only check, backup on overwrite).
+"""
+
+from __future__ import annotations
+
+import csv as csv_lib
+import io
+import logging
+import re
+from typing import Any
+from xml.sax import saxutils
+
+from backend.services.errors import ServiceError
+from backend.services.mutations import save_raw_file
+from backend.tools.context import ToolContext, ToolError, ToolRisk
+from backend.tools.registry import tool
+from backend.tools.schemas import CsvInput, DocxInput, PdfInput, SpreadsheetInput
+
+logger = logging.getLogger("obsigate.tools.documents")
+
+MAX_PDF_CHARS = 200_000
+MAX_ROWS = 5_000
+
+
+def _save(vault: str, path: str, content: bytes, overwrite: bool) -> dict[str, Any]:
+    """Shared save helper (maps ServiceError to ToolError)."""
+    try:
+        return save_raw_file(vault, path, content, overwrite=overwrite, allow_docs=True)
+    except ServiceError as e:
+        raise ToolError(e.message, code=e.code, details=e.details) from e
+
+
+def _check_rows(rows: list[list[Any]]) -> None:
+    if not rows:
+        raise ToolError("Aucune ligne fournie", code="invalid_arguments")
+    if len(rows) > MAX_ROWS:
+        raise ToolError(
+            f"Trop de lignes ({len(rows)} > {MAX_ROWS})", code="invalid_arguments"
+        )
+
+
+def _check_extension(path: str, expected: str) -> str:
+    """Enforce the document extension; return the normalized path."""
+    path = (path or "").strip()
+    if not path.lower().endswith(expected):
+        raise ToolError(
+            f"Extension attendue : {expected}", code="invalid_arguments"
+        )
+    return path
+
+
+@tool(
+    name="create_xlsx",
+    description=(
+        "Create an .xlsx spreadsheet in a vault from rows of cell values "
+        "(first row = header). Use for tables, budgets, checklists the user "
+        "asked to turn into an Excel file."
+    ),
+    input_model=SpreadsheetInput,
+    risk=ToolRisk.WRITE,
+    requires_vault=True,
+)
+def create_xlsx(ctx: ToolContext, params: SpreadsheetInput) -> dict[str, Any]:
+    """Build the workbook with openpyxl and save it into the vault."""
+    from openpyxl import Workbook
+
+    _check_rows(params.rows)
+    path = _check_extension(params.path, ".xlsx")
+    wb = Workbook()
+    ws = wb.active
+    ws.title = params.sheet_name[:31] or "Feuille1"
+    for row in params.rows:
+        ws.append(list(row))
+    buffer = io.BytesIO()
+    wb.save(buffer)
+    return _save(params.vault, path, buffer.getvalue(), params.overwrite)
+
+
+@tool(
+    name="create_docx",
+    description=(
+        "Create a .docx Word document in a vault from an optional title and "
+        "ordered paragraphs. Use for letters, reports, structured drafts."
+    ),
+    input_model=DocxInput,
+    risk=ToolRisk.WRITE,
+    requires_vault=True,
+)
+def create_docx(ctx: ToolContext, params: DocxInput) -> dict[str, Any]:
+    """Build the document with python-docx and save it into the vault."""
+    from docx import Document
+
+    if not params.paragraphs:
+        raise ToolError("Aucun paragraphe fourni", code="invalid_arguments")
+    path = _check_extension(params.path, ".docx")
+    doc = Document()
+    if params.title.strip():
+        doc.add_heading(params.title.strip(), level=1)
+    for paragraph in params.paragraphs:
+        doc.add_paragraph(paragraph)
+    buffer = io.BytesIO()
+    doc.save(buffer)
+    return _save(params.vault, path, buffer.getvalue(), params.overwrite)
+
+
+@tool(
+    name="create_csv",
+    description=(
+        "Create a .csv file in a vault from rows of cell values (first row = "
+        "header). Use for flat data exports, simple tables."
+    ),
+    input_model=CsvInput,
+    risk=ToolRisk.WRITE,
+    requires_vault=True,
+)
+def create_csv(ctx: ToolContext, params: CsvInput) -> dict[str, Any]:
+    """Serialize the rows and save the CSV into the vault."""
+    _check_rows(params.rows)
+    path = _check_extension(params.path, ".csv")
+    delimiter = params.delimiter if params.delimiter in (",", ";", "\t") else ","
+    buffer = io.StringIO()
+    writer = csv_lib.writer(buffer, delimiter=delimiter, lineterminator="\n")
+    writer.writerows(params.rows)
+    return _save(params.vault, path, buffer.getvalue().encode("utf-8"), params.overwrite)
+
+
+_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$")
+
+
+def _markdown_to_flowables(content: str) -> list[tuple[str, str]]:
+    """Split markdown-ish content into (style, text) blocks for reportlab."""
+    blocks: list[tuple[str, str]] = []
+    for raw_line in content.splitlines():
+        line = raw_line.rstrip()
+        if not line.strip():
+            continue
+        heading = _HEADING_RE.match(line)
+        if heading:
+            blocks.append((f"H{min(3, len(heading.group(1)))}", heading.group(2).strip()))
+        else:
+            blocks.append(("P", line.strip()))
+    return blocks
+
+
+@tool(
+    name="create_pdf",
+    description=(
+        "Create a .pdf document in a vault from markdown-ish content "
+        "(# headings + paragraphs). Use for printable deliverables."
+    ),
+    input_model=PdfInput,
+    risk=ToolRisk.WRITE,
+    requires_vault=True,
+)
+def create_pdf(ctx: ToolContext, params: PdfInput) -> dict[str, Any]:
+    """Render the content with reportlab and save the PDF into the vault."""
+    from reportlab.lib.pagesizes import A4
+    from reportlab.lib.styles import getSampleStyleSheet
+    from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer
+
+    path = _check_extension(params.path, ".pdf")
+    text = params.content[:MAX_PDF_CHARS]
+    styles = getSampleStyleSheet()
+    style_map = {
+        "P": styles["BodyText"],
+        "H1": styles["Heading1"],
+        "H2": styles["Heading2"],
+        "H3": styles["Heading3"],
+    }
+    buffer = io.BytesIO()
+    doc = SimpleDocTemplate(buffer, pagesize=A4, title=params.title[:200])
+    story: list[Any] = [Paragraph(saxutils.escape(params.title[:300]), styles["Title"])]
+    for style, line in _markdown_to_flowables(text):
+        story.append(Spacer(1, 4))
+        story.append(Paragraph(saxutils.escape(line), style_map[style]))
+    doc.build(story)
+    return _save(params.vault, path, buffer.getvalue(), params.overwrite)

backend/tools/labels.py

@@ -47,6 +47,14 @@ _STEP_LABELS: dict[str, tuple[str, str | None]] = {
     "restore_backup": ("backup_restore", "path"),
     "web_search": ("web_search", "query"),
     "fetch_url": ("fetch_url", "url"),
+    "crawl_site": ("crawl", "url"),
+    "git_list_repos": ("git_repos", "provider"),
+    "git_search_issues": ("git_issues", "query"),
+    "git_get_file": ("git_file", "path"),
+    "create_xlsx": ("xlsx_create", "path"),
+    "create_docx": ("docx_create", "path"),
+    "create_csv": ("csv_create", "path"),
+    "create_pdf": ("pdf_create", "path"),
 }
 
 GENERIC_KEY = "generic"

backend/tools/schemas.py

@@ -251,6 +251,90 @@ class FetchUrlInput(BaseModel):
     """Fetch one public web page and return its readable text."""
 
     url: str = Field(..., description="Absolute http(s) URL of a public page")
+    render: bool = Field(
+        False,
+        description="Render JavaScript with the optional Playwright worker (dynamic SPA pages)",
+    )
+
+
+class CrawlSiteInput(BaseModel):
+    """Crawl a small public site (same-host only) and save a digest into a vault."""
+
+    url: str = Field(..., description="Absolute http(s) URL where the crawl starts")
+    vault: str = Field(..., description="Vault name")
+    path: str = Field(..., description="Vault-relative path of the digest file to write (.md)")
+    max_pages: int = Field(5, ge=1, le=20, description="Maximum number of pages to crawl")
+
+
+class GitProviderInput(BaseModel):
+    """Base fields for connected-source tools (Gitea / GitHub)."""
+
+    provider: str = Field(..., description="'gitea' (OBSIGATE_GITEA_URL) or 'github'")
+    repo: str = Field("", description="Optional 'owner/name' repository filter")
+    limit: int = Field(20, ge=1, le=50, description="Maximum number of entries")
+
+
+class GitSearchIssuesInput(BaseModel):
+    """Search issues/pull requests on a connected Gitea or GitHub instance."""
+
+    provider: str = Field(..., description="'gitea' or 'github'")
+    query: str = Field(..., min_length=1, description="Search keywords")
+    repo: str = Field("", description="Optional 'owner/name' scope (empty = instance-wide)")
+    state: str = Field("open", description="'open' or 'closed'")
+    limit: int = Field(10, ge=1, le=20, description="Maximum number of issues")
+
+
+class GitGetFileInput(BaseModel):
+    """Read a file from a connected Gitea or GitHub repository."""
+
+    provider: str = Field(..., description="'gitea' or 'github'")
+    repo: str = Field(..., description="'owner/name' repository")
+    path: str = Field(..., description="Repository-relative file path")
+    ref: str = Field("", description="Optional branch/tag/commit (empty = default branch)")
+
+
+class SpreadsheetInput(BaseModel):
+    """Create an .xlsx spreadsheet in a vault from rows of cells."""
+
+    vault: str = Field(..., description="Vault name")
+    path: str = Field(..., description="Vault-relative path of the file to write (.xlsx)")
+    rows: list[list[str | int | float | bool | None]] = Field(
+        ..., description="Rows of cell values (first row = header)"
+    )
+    sheet_name: str = Field("Feuille1", description="Worksheet name")
+    overwrite: bool = Field(True, description="Replace an existing file (with backup)")
+
+
+class DocxInput(BaseModel):
+    """Create a .docx Word document in a vault from paragraphs."""
+
+    vault: str = Field(..., description="Vault name")
+    path: str = Field(..., description="Vault-relative path of the file to write (.docx)")
+    title: str = Field("", description="Optional document title (heading 1)")
+    paragraphs: list[str] = Field(..., description="Paragraph texts, in order")
+    overwrite: bool = Field(True, description="Replace an existing file (with backup)")
+
+
+class CsvInput(BaseModel):
+    """Create a .csv file in a vault from rows of cells."""
+
+    vault: str = Field(..., description="Vault name")
+    path: str = Field(..., description="Vault-relative path of the file to write (.csv)")
+    rows: list[list[str | int | float | bool | None]] = Field(
+        ..., description="Rows of cell values (first row = header)"
+    )
+    delimiter: str = Field(",", description="Field separator (',' ';' '\\t')")
+    overwrite: bool = Field(True, description="Replace an existing file (with backup)")
+
+
+class PdfInput(BaseModel):
+    """Create a .pdf document in a vault from markdown-ish content."""
+
+    vault: str = Field(..., description="Vault name")
+    path: str = Field(..., description="Vault-relative path of the file to write (.pdf)")
+    title: str = Field("Document", description="Document title")
+    content: str = Field(..., description="Content (headings with #/##, then paragraphs)")
+    overwrite: bool = Field(True, description="Replace an existing file (with backup)")
 
 
 class ToolResult(BaseModel):

backend/tools/web.py

@@ -8,6 +8,16 @@ Phase 1 of the documented web-toolset roadmap:
   answering « je n'ai pas accès à internet ».
 * ``fetch_url``   — retrieve a public web page and return readable text.
 
+Phase 2 (#92) additions:
+
+* keyed providers — Tavily, Brave Search, SerpAPI and Exa are used first when
+  their API key is configured (env, injected by Infisical in production);
+* SQLite cache — search/fetch results are cached with a TTL
+  (:mod:`backend.tools.webcache`);
+* retry with backoff — transient network errors get one extra attempt;
+* dynamic rendering — ``fetch_url(render=True)`` uses an isolated Playwright
+  worker (optional dependency, graceful degradation).
+
 All are READ-risk tools (no confirmation), rate-limited through the shared
 registry, SSRF-guarded (scheme + private-address rejection), and size-capped.
 
@@ -16,6 +26,14 @@ Configuration (environment):
 * ``OBSIGATE_WEB_TIMEOUT``   — seconds, default 10
 * ``OBSIGATE_WEB_FALLBACK``  — ``0``/``false`` disables the keyless HTML
   fallbacks (SearXNG only), default enabled
+* ``OBSIGATE_TAVILY_API_KEY`` / ``OBSIGATE_BRAVE_API_KEY`` /
+  ``OBSIGATE_SERPAPI_API_KEY`` / ``OBSIGATE_EXA_API_KEY`` — optional keyed
+  providers, tried before SearXNG when set
+* ``OBSIGATE_WEB_PROVIDERS`` — optional comma-separated provider order
+  (e.g. ``brave,searxng``); keyed providers without a key are skipped
+* ``OBSIGATE_WEB_RETRY``     — extra attempts for transient network errors
+  (default 1)
+* ``OBSIGATE_WEB_CACHE_TTL`` — cache TTL seconds, ``0`` disables (default 900)
 """
 
 from __future__ import annotations
@@ -28,12 +46,14 @@ import logging
 import os
 import re
 import socket
+import time
 from collections.abc import Callable
 from typing import Any
 from urllib.parse import parse_qs, urlparse
 
 import httpx
 
+from backend.tools import webcache
 from backend.tools.context import ToolError, ToolRisk, ToolScope
 from backend.tools.registry import tool
 from backend.tools.schemas import FetchUrlInput, WebSearchInput
@@ -48,6 +68,7 @@ WEB_FALLBACK_ENABLED = os.environ.get("OBSIGATE_WEB_FALLBACK", "1").strip().lowe
     "no",
     "off",
 }
+WEB_RETRY_ATTEMPTS = int(os.environ.get("OBSIGATE_WEB_RETRY", "1"))
 USER_AGENT = "ObsiGateAssistant/1.0 (+self-hosted vault AI)"
 # Search engines reject non-browser agents on their public HTML endpoints.
 BROWSER_UA = (
@@ -163,6 +184,115 @@ def _result(
     }
 
 
+def _with_retry(call: Callable[[], Any]) -> Any:
+    """Run *call* with one extra attempt on transient network errors.
+
+    House-made backoff (the roadmap's « tenacity ou boucle maison »): DNS
+    blips and rate-limit hiccups are the common failure mode, and a single
+    retry keeps the fallback chain from being consumed too early.
+    """
+    for attempt in range(1 + max(0, WEB_RETRY_ATTEMPTS)):
+        try:
+            return call()
+        except httpx.TransportError:
+            if attempt >= max(0, WEB_RETRY_ATTEMPTS):
+                raise
+            time.sleep(0.2 * (attempt + 1))
+    raise RuntimeError("unreachable")  # pragma: no cover
+
+
+def _env_key(name: str) -> str:
+    """Read an API key from the environment (Infisical-injected in prod)."""
+    return os.environ.get(name, "").strip()
+
+
+def _search_tavily(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]:
+    """Tavily Search API (agent-oriented results, key required)."""
+    resp = httpx.post(
+        "https://api.tavily.com/search",
+        json={
+            "api_key": _env_key("OBSIGATE_TAVILY_API_KEY"),
+            "query": query,
+            "max_results": params.max_results,
+            "search_depth": "basic",
+            "include_answer": False,
+        },
+        headers={"User-Agent": USER_AGENT},
+        timeout=WEB_TIMEOUT,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    return [
+        _result(item.get("title") or "", item.get("url") or "", item.get("content") or "")
+        for item in (data.get("results") or [])
+    ], []
+
+
+def _search_brave(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]:
+    """Brave Search API (key required)."""
+    resp = httpx.get(
+        "https://api.search.brave.com/res/v1/web/search",
+        params={"q": query, "count": params.max_results, "safesearch": "moderate"},
+        headers={
+            "X-Subscription-Id": _env_key("OBSIGATE_BRAVE_API_KEY"),
+            "Accept": "application/json",
+            "User-Agent": USER_AGENT,
+        },
+        timeout=WEB_TIMEOUT,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    return [
+        _result(item.get("title") or "", item.get("url") or "", item.get("description") or "")
+        for item in ((data.get("web") or {}).get("results") or [])
+    ], []
+
+
+def _search_serpapi(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]:
+    """SerpAPI (Google SERP, key required)."""
+    resp = httpx.get(
+        "https://serpapi.com/search",
+        params={"q": query, "api_key": _env_key("OBSIGATE_SERPAPI_API_KEY"),
+                "num": params.max_results},
+        headers={"User-Agent": USER_AGENT},
+        timeout=WEB_TIMEOUT,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    return [
+        _result(item.get("title") or "", item.get("link") or "", item.get("snippet") or "")
+        for item in (data.get("organic_results") or [])
+    ], []
+
+
+def _search_exa(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]:
+    """Exa neural search (key required)."""
+    resp = httpx.post(
+        "https://api.exa.ai/search",
+        json={"query": query, "numResults": params.max_results},
+        headers={
+            "x-api-key": _env_key("OBSIGATE_EXA_API_KEY"),
+            "User-Agent": USER_AGENT,
+        },
+        timeout=WEB_TIMEOUT,
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    return [
+        _result(item.get("title") or "", item.get("url") or "", (item.get("text") or "")[:600])
+        for item in (data.get("results") or [])
+    ], []
+
+
+# Keyed providers: name -> (implementation, API key env var)
+_KEYED_PROVIDERS: dict[str, tuple[_Provider, str]] = {
+    "tavily": (_search_tavily, "OBSIGATE_TAVILY_API_KEY"),
+    "brave": (_search_brave, "OBSIGATE_BRAVE_API_KEY"),
+    "serpapi": (_search_serpapi, "OBSIGATE_SERPAPI_API_KEY"),
+    "exa": (_search_exa, "OBSIGATE_EXA_API_KEY"),
+}
+
+
 def _search_searxng(
     query: str, params: WebSearchInput
 ) -> tuple[list[dict[str, Any]], list[str]]:
@@ -288,8 +418,22 @@ _Provider = Callable[[str, WebSearchInput], "tuple[list[dict[str, Any]], list[st
 
 
 def _provider_chain() -> list[tuple[str, _Provider]]:
-    """Ordered providers: self-hosted meta-search first, then keyless fallbacks."""
-    chain: list[tuple[str, _Provider]] = [("searxng", _search_searxng)]
+    """Ordered providers: keyed APIs first, then self-hosted, then keyless.
+
+    ``OBSIGATE_WEB_PROVIDERS`` (comma-separated) overrides the default order;
+    unknown names are ignored and keyed providers without their key are skipped.
+    """
+    chain: list[tuple[str, _Provider]] = []
+    configured = [
+        name.strip().lower()
+        for name in os.environ.get("OBSIGATE_WEB_PROVIDERS", "").split(",")
+        if name.strip()
+    ]
+    for name in configured or list(_KEYED_PROVIDERS):
+        entry = _KEYED_PROVIDERS.get(name)
+        if entry and _env_key(entry[1]):
+            chain.append((name, entry[0]))
+    chain.append(("searxng", _search_searxng))
     if WEB_FALLBACK_ENABLED:
         chain.append(("duckduckgo", _search_duckduckgo))
         chain.append(("bing", _search_bing))
@@ -313,6 +457,17 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
     if not query:
         raise ToolError("Requête vide", code="invalid_arguments")
 
+    key = webcache.cache_key("search", {
+        "q": query,
+        "max_results": params.max_results,
+        "category": params.category,
+        "language": params.language,
+        "page": params.page,
+    })
+    cached = webcache.cache_get(key)
+    if cached is not None:
+        return {**cached, "cached": True}
+
     attempts: list[str] = []
     unresponsive: list[str] = []
     reachable = False
@@ -320,8 +475,12 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
 
     for name, provider in _provider_chain():
         attempts.append(name)
+
+        def _attempt(p: _Provider = provider) -> tuple[list[dict[str, Any]], list[str]]:
+            return p(query, params)
+
         try:
-            results, engines = provider(query, params)
+            results, engines = _with_retry(_attempt)
         except (httpx.HTTPError, ValueError, AttributeError) as e:
             logger.warning("web_search provider %s failed: %s", name, e)
             last_error = e
@@ -338,6 +497,7 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
             }
             if unresponsive:
                 payload["unresponsive_engines"] = unresponsive[:8]
+            webcache.cache_set(key, payload)
             return payload
 
     if not reachable:
@@ -378,6 +538,20 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]:
 def fetch_url(ctx, params: FetchUrlInput) -> dict[str, Any]:
     """Retrieve one page, guard against SSRF, and extract its text."""
     url = _assert_public_http_url(params.url.strip())
+    key = webcache.cache_key("fetch", {"url": url, "render": params.render})
+    cached = webcache.cache_get(key)
+    if cached is not None:
+        return {**cached, "cached": True}
+
+    if params.render:
+        # Dynamic pages (SPA/React): delegated to the isolated Playwright
+        # worker; the browser dependency stays optional (graceful error).
+        from backend.tools.webrender import render_page
+
+        payload = render_page(url)
+        webcache.cache_set(key, payload)
+        return payload
+
     try:
         # Follow redirects manually so every hop is re-checked against the
         # private-address SSRF guard (a public page can redirect to 127.0.0.1).
@@ -414,10 +588,12 @@ def fetch_url(ctx, params: FetchUrlInput) -> dict[str, Any]:
     title_match = re.search(r"<title[^>]*>(.*?)</title>", raw, re.IGNORECASE | re.DOTALL)
     title = html_lib.unescape(title_match.group(1)).strip()[:300] if title_match else ""
     text = _html_to_text(raw)[:MAX_TEXT_CHARS]
-    return {
+    payload = {
         "url": str(resp.url),
         "status": resp.status_code,
         "title": title,
         "text": text,
         "truncated": len(raw) > MAX_TEXT_CHARS,
     }
+    webcache.cache_set(key, payload)
+    return payload

backend/tools/webcache.py

@@ -0,0 +1,138 @@
+"""SQLite cache for web tool results (search results, fetched pages).
+
+Phase 2 of the web-toolset roadmap (« Transverse »): repeated web searches and
+page fetches (common in agent loops, where the model re-reads a source) must
+not hammer the providers. Results are cached in a dedicated SQLite table with
+a TTL; the cache is best-effort — any error silently disables it so a broken
+database file never takes the assistant down.
+
+Configuration (environment):
+* ``OBSIGATE_DATA_DIR``        — base data directory (default ``data``)
+* ``OBSIGATE_WEB_CACHE_PATH``  — explicit cache file override
+* ``OBSIGATE_WEB_CACHE_TTL``   — seconds, ``0`` disables the cache (default 900)
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import sqlite3
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("obsigate.tools.webcache")
+
+DEFAULT_TTL_SECONDS = 900
+_schema_ready = False
+_write_lock = threading.Lock()
+
+
+def ttl_seconds() -> float:
+    """Configured TTL in seconds (``0`` = cache disabled)."""
+    return float(os.environ.get("OBSIGATE_WEB_CACHE_TTL", str(DEFAULT_TTL_SECONDS)))
+
+
+def _cache_path() -> Path:
+    override = os.environ.get("OBSIGATE_WEB_CACHE_PATH", "").strip()
+    if override:
+        return Path(override)
+    return Path(os.environ.get("OBSIGATE_DATA_DIR", "data")) / "web_cache.sqlite3"
+
+
+def _connect() -> sqlite3.Connection:
+    """Open (and lazily create) the cache database."""
+    global _schema_ready
+    path = _cache_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(path, timeout=5, check_same_thread=False)
+    if not _schema_ready:
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS web_cache ("
+            "key TEXT PRIMARY KEY, value TEXT NOT NULL, created REAL NOT NULL)"
+        )
+        conn.commit()
+        _schema_ready = True
+    return conn
+
+
+def cache_key(prefix: str, payload: dict[str, Any]) -> str:
+    """Deterministic cache key from a prefix and the normalized arguments."""
+    raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str)
+    digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32]
+    return f"{prefix}:{digest}"
+
+
+def cache_get(key: str) -> Any | None:
+    """Return the cached payload for *key*, or ``None`` (miss/expiry/disabled)."""
+    if ttl_seconds() <= 0:
+        return None
+    try:
+        conn = _connect()
+        row = conn.execute(
+            "SELECT value, created FROM web_cache WHERE key = ?", (key,)
+        ).fetchone()
+        conn.close()
+    except sqlite3.Error as e:
+        logger.warning("web cache read failed (%s): %s", key, e)
+        return None
+    if row is None:
+        return None
+    value, created = row
+    if time.time() - float(created) > ttl_seconds():
+        return None
+    try:
+        return json.loads(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def cache_set(key: str, value: Any) -> None:
+    """Store *value* under *key* (best effort, never raises)."""
+    if ttl_seconds() <= 0:
+        return
+    try:
+        with _write_lock:
+            conn = _connect()
+            conn.execute(
+                "INSERT INTO web_cache (key, value, created) VALUES (?, ?, ?) "
+                "ON CONFLICT(key) DO UPDATE SET value = excluded.value, created = excluded.created",
+                (key, json.dumps(value, ensure_ascii=False, default=str), time.time()),
+            )
+            conn.commit()
+            conn.close()
+    except sqlite3.Error as e:
+        logger.warning("web cache write failed (%s): %s", key, e)
+
+
+def purge_expired() -> int:
+    """Delete expired rows; return the number of removed entries (maintenance)."""
+    try:
+        conn = _connect()
+        cursor = conn.execute(
+            "DELETE FROM web_cache WHERE created < ?", (time.time() - ttl_seconds(),)
+        )
+        conn.commit()
+        deleted = cursor.rowcount
+        conn.close()
+        return int(deleted)
+    except sqlite3.Error as e:
+        logger.warning("web cache purge failed: %s", e)
+        return 0
+
+
+def clear_cache() -> int:
+    """Drop every cached entry (tests / admin); returns the number of rows."""
+    try:
+        conn = _connect()
+        cursor = conn.execute("DELETE FROM web_cache")
+        conn.commit()
+        deleted = cursor.rowcount
+        conn.close()
+        return int(deleted)
+    except sqlite3.Error as e:
+        logger.warning("web cache clear failed: %s", e)
+        return 0

backend/tools/webrender.py

@@ -0,0 +1,100 @@
+"""Dynamic page rendering (Playwright) — ``fetch_url(render=True)``.
+
+Static pages are fetched with httpx inside :mod:`backend.tools.web`. Dynamic
+pages (SPA/React, JS-loaded content) need a real browser engine; this module
+runs one Playwright call inside a dedicated worker thread so browser
+crashes/timeouts never take over the tool layer, and the heavyweight
+dependency stays optional:
+
+* not installed → ``ToolError(code="playwright_unavailable")`` with a clear
+  message (the assistant explains the limitation instead of hanging);
+* installed → ``pip install playwright && playwright install chromium``.
+
+The SSRF guard (scheme + private-address rejection) is applied before the
+browser navigates. Note: unlike the httpx path, internal redirects performed
+by the browser engine are not re-checked hop by hop.
+"""
+
+from __future__ import annotations
+
+import html as html_lib
+import logging
+import re
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+from backend.tools.context import ToolError
+from backend.tools.web import (
+    MAX_TEXT_CHARS,
+    USER_AGENT,
+    _assert_public_http_url,
+    _html_to_text,
+)
+
+logger = logging.getLogger("obsigate.tools.webrender")
+
+# One worker: browser automation is serialized on purpose (one Chromium at a
+# time keeps memory predictable on small hosts).
+_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="obsigate-playwright")
+GOTO_TIMEOUT_MS = 20_000
+
+
+def _playwright_available() -> bool:
+    try:
+        import playwright  # noqa: F401
+    except ImportError:
+        return False
+    return True
+
+
+def _render_in_worker(url: str) -> dict[str, Any]:
+    """Synchronous Playwright render — runs in the dedicated worker thread."""
+    from playwright.sync_api import sync_playwright
+
+    status = 0
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        try:
+            page = browser.new_page(user_agent=USER_AGENT)
+            response = page.goto(url, wait_until="networkidle", timeout=GOTO_TIMEOUT_MS)
+            if response is not None:
+                status = response.status
+            raw = page.content()
+            title = html_lib.unescape(page.title() or "").strip()
+            text = _html_to_text(raw)[:MAX_TEXT_CHARS]
+        finally:
+            browser.close()
+    title = re.sub(r"\s+", " ", title)[:300]
+    return {
+        "url": url,
+        "status": status,
+        "title": title,
+        "text": text,
+        "rendered": True,
+        "truncated": len(raw) > MAX_TEXT_CHARS,
+    }
+
+
+def render_page(url: str) -> dict[str, Any]:
+    """Render *url* (JavaScript included) and return readable text.
+
+    Raises:
+        ToolError: ``playwright_unavailable`` when the optional dependency is
+            missing, ``render_unavailable`` when the render itself failed.
+    """
+    _assert_public_http_url(url)
+    if not _playwright_available():
+        raise ToolError(
+            "Rendu dynamique indisponible : Playwright n'est pas installé "
+            "(pip install playwright && playwright install chromium).",
+            code="playwright_unavailable",
+        )
+    try:
+        return _executor.submit(_render_in_worker, url).result(timeout=GOTO_TIMEOUT_MS / 1000 + 40)
+    except ToolError:
+        raise
+    except Exception as e:
+        logger.warning("render_page failed for %s: %s", url, e)
+        raise ToolError(
+            "Le rendu dynamique de la page a échoué.", code="render_unavailable"
+        ) from e