From 6a58a59a11bd0443bf23c8a6820c3dbea7f02f36 Mon Sep 17 00:00:00 2001 From: Bruno Charest Date: Thu, 17 Sep 2026 11:52:00 -0400 Subject: [PATCH] feat(ai): ecosysteme d'outils phase 2 - recherche a cle, cache/retry, Playwright, crawl, Gitea/GitHub, documents (#92) --- .env.example | 16 ++ CHANGELOG.md | 31 +++- README.fr.md | 12 +- README.md | 12 +- VERSION | 2 +- backend/requirements.txt | 3 + backend/services/mutations.py | 12 +- backend/tools/api.py | 3 + backend/tools/connected.py | 239 +++++++++++++++++++++++++++++ backend/tools/crawler.py | 197 ++++++++++++++++++++++++ backend/tools/documents.py | 188 +++++++++++++++++++++++ backend/tools/labels.py | 8 + backend/tools/schemas.py | 84 ++++++++++ backend/tools/web.py | 184 +++++++++++++++++++++- backend/tools/webcache.py | 138 +++++++++++++++++ backend/tools/webrender.py | 100 ++++++++++++ desktop/Cargo.lock | 2 +- desktop/Cargo.toml | 2 +- desktop/tauri.conf.json | 2 +- docs/ROADMAP.md | 25 +-- docs/features/ai-tools-roadmap.md | 13 +- frontend/locales/en.json | 8 + frontend/locales/fr.json | 8 + package.json | 2 +- tests/conftest.py | 17 ++ tests/test_connected_tools.py | 230 +++++++++++++++++++++++++++ tests/test_crawler.py | 135 ++++++++++++++++ tests/test_document_tools.py | 160 +++++++++++++++++++ tests/test_web_cache.py | 82 ++++++++++ tests/test_web_search_providers.py | 155 +++++++++++++++++++ tests/test_webrender.py | 73 +++++++++ 31 files changed, 2102 insertions(+), 41 deletions(-) create mode 100644 backend/tools/connected.py create mode 100644 backend/tools/crawler.py create mode 100644 backend/tools/documents.py create mode 100644 backend/tools/webcache.py create mode 100644 backend/tools/webrender.py create mode 100644 tests/test_connected_tools.py create mode 100644 tests/test_crawler.py create mode 100644 tests/test_document_tools.py create mode 100644 tests/test_web_cache.py create mode 100644 tests/test_web_search_providers.py create mode 100644 tests/test_webrender.py diff --git a/.env.example b/.env.example index c342f1d..b159929 100644 --- a/.env.example +++ b/.env.example @@ -74,3 +74,19 @@ DEEPSEEK_MODEL=deepseek-chat # Chaîne de repli sans clé (DuckDuckGo puis Bing) si SearXNG ne remonte rien # OBSIGATE_WEB_FALLBACK=1 # OBSIGATE_WEB_TIMEOUT=10 +# Fournisseurs à clé (#92), essayés avant SearXNG — injecter via Infisical en prod +# OBSIGATE_TAVILY_API_KEY= +# OBSIGATE_BRAVE_API_KEY= +# OBSIGATE_SERPAPI_API_KEY= +# OBSIGATE_EXA_API_KEY= +# Ordre des fournisseurs (sinon : clés présentes puis SearXNG puis replis) +# OBSIGATE_WEB_PROVIDERS=brave,searxng +# Réessais réseau (backoff maison) + cache SQLite des résultats web +# OBSIGATE_WEB_RETRY=1 +# OBSIGATE_WEB_CACHE_TTL=900 # secondes ; 0 = cache désactivé +# Rendu dynamique (pages SPA) — dépendance optionnelle : +# pip install playwright && playwright install chromium +# ── Assistant IA — sources connectées (Gitea / GitHub) ── +# OBSIGATE_GITEA_URL=https://git.example.net +# OBSIGATE_GITEA_TOKEN= +# OBSIGATE_GITHUB_TOKEN= diff --git a/CHANGELOG.md b/CHANGELOG.md index 38bd118..4a96661 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Format basé sur [Keep a Changelog](https://keepachangelog.com/fr/1.1.0/), et [Semantic Versioning](https://semver.org/spec/v2.0.0.html). > **En cours de développement** : les changements à venir sont listés dans la section -> [Unreleased](#unreleased). La dernière version livrée est **2.9.1**. +> [Unreleased](#unreleased). La dernière version livrée est **2.10.0**. --- @@ -14,6 +14,35 @@ et [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --- +## [2.10.0] — 2026-09-17 + +### Ajouté + +- **#92 — Assistant IA : écosystème d'outils phase 2 (web étendu, sources connectées, documents)** : + - **Recherche web à clé** : fournisseurs optionnels essayés avant SearXNG — Tavily, Brave + Search, SerpAPI (Google) et Exa (`OBSIGATE_TAVILY_API_KEY`, `OBSIGATE_BRAVE_API_KEY`, + `OBSIGATE_SERPAPI_API_KEY`, `OBSIGATE_EXA_API_KEY`), avec ordre personnalisable via + `OBSIGATE_WEB_PROVIDERS`. + - **Transverse** : réessais réseau avec backoff maison (`OBSIGATE_WEB_RETRY`) et cache SQLite + des résultats web avec TTL (`OBSIGATE_WEB_CACHE_TTL`, 900 s par défaut, `OBSIGATE_WEB_CACHE_PATH`). + - **Rendu dynamique** : `fetch_url(render=True)` délègue les pages SPA à un worker Playwright + isolé (dépendance optionnelle, dégradation propre si non installée). + - **Crawl de site** : `crawl_site` (WRITE, confirmation) capture jusqu'à 20 pages d'un même + hôte et enregistre un condensé Markdown dans un vault. + - **Sources connectées** : Gitea (`OBSIGATE_GITEA_URL`/`OBSIGATE_GITEA_TOKEN`) et GitHub + (`OBSIGATE_GITHUB_TOKEN`) — `git_list_repos`, `git_search_issues`, `git_get_file` (READ, + rate-limités, audités). Les drives cloud (Google Drive / OneDrive) restent orientés serveur + MCP externe (#79), conformément à la feuille de route. + - **Production de documents** : `create_xlsx` (openpyxl), `create_docx` (python-docx), + `create_csv`, `create_pdf` (reportlab) — outils WRITE avec confirmation et sauvegarde + dans le vault (backup avant écrasement). + - Chaque outil : libellé `labels.py` + clés i18n `ai.step.*` FR/EN + tests unitaires mockés + (httpx). Dépendances ajoutées : `openpyxl`, `python-docx`, `reportlab`. + Fichiers : `backend/tools/{webcache,webrender,connected,crawler,documents,web,schemas,labels}.py`, + `backend/services/mutations.py`, `tests/test_{web_cache,web_search_providers,webrender,connected_tools,document_tools,crawler}.py`. + +--- + ## [2.9.1] — 2026-09-17 ### Corrigé diff --git a/README.fr.md b/README.fr.md index 0daff3e..12211f5 100644 --- a/README.fr.md +++ b/README.fr.md @@ -4,7 +4,7 @@ **Porte d'entrée web ultra-léger pour vos vaults Obsidian** — Accédez, naviguez et recherchez dans toutes vos notes Obsidian depuis n'importe quel appareil via une interface web moderne et responsive. -[![Version](https://img.shields.io/badge/Version-2.9.1-blue.svg)]() +[![Version](https://img.shields.io/badge/Version-2.10.0-blue.svg)]() [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Docker](https://img.shields.io/badge/Docker-Ready-blue.svg)](https://www.docker.com/) [![Python](https://img.shields.io/badge/Python-3.11+-green.svg)](https://www.python.org/) @@ -282,6 +282,12 @@ Un compte **admin** connecté voit une icône 🛡️ dans le header : liste, cr | `OBSIGATE_WEBHOOK_ALLOW_PRIVATE` | Autoriser les webhooks vers des adresses privées/boucle | `false` | | `OBSIGATE_PDF_MAX_SIZE_MB` | Taille max des PDF extraits (text indexation) | `50` | | `OBSIGATE_PDF_EXTRACT_TIMEOUT` | Timeout extraction PDF (secondes) | `30` | +| `OBSIGATE_TAVILY_API_KEY` / `OBSIGATE_BRAVE_API_KEY` / `OBSIGATE_SERPAPI_API_KEY` / `OBSIGATE_EXA_API_KEY` | Fournisseurs de recherche web à clé (essayés avant SearXNG) | — | +| `OBSIGATE_WEB_PROVIDERS` | Ordre des fournisseurs de recherche (ex. `brave,searxng`) | — | +| `OBSIGATE_WEB_RETRY` | Réessais réseau des outils web (backoff maison) | `1` | +| `OBSIGATE_WEB_CACHE_TTL` | Durée du cache SQLite des résultats web (secondes, `0` = off) | `900` | +| `OBSIGATE_GITEA_URL` / `OBSIGATE_GITEA_TOKEN` | Source connectée Gitea (outil `git_list_repos`…) | — | +| `OBSIGATE_GITHUB_TOKEN` | Jeton GitHub (outil `git_list_repos`…) | — | ### Volume pour la persistance @@ -916,8 +922,8 @@ Ce projet est sous licence **MIT** — voir le fichier [LICENSE](LICENSE) pour l ## 📝 Changelog -Consultez le [CHANGELOG.md](./CHANGELOG.md) pour l'historique complet de toutes les versions (v1.0.0 → v2.9.1). +Consultez le [CHANGELOG.md](./CHANGELOG.md) pour l'historique complet de toutes les versions (v1.0.0 → v2.10.0). --- -*Projet : ObsiGate | Version : 2.9.1 | Dernière mise à jour : Juin 2026* +*Projet : ObsiGate | Version : 2.10.0 | Dernière mise à jour : Juin 2026* diff --git a/README.md b/README.md index 3b8ff52..65546a8 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **Ultra-light web gateway for your Obsidian vaults** — Access, browse, and search all your Obsidian notes from any device via a modern, responsive web interface. -[![Version](https://img.shields.io/badge/Version-2.9.1-blue.svg)]() +[![Version](https://img.shields.io/badge/Version-2.10.0-blue.svg)]() [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Docker](https://img.shields.io/badge/Docker-Ready-blue.svg)](https://www.docker.com/) [![Python](https://img.shields.io/badge/Python-3.11+-green.svg)](https://www.python.org/) @@ -320,6 +320,12 @@ When an **admin** account is logged in, a 🛡️ icon appears in the header. Cl | `OBSIGATE_WEBHOOK_ALLOW_PRIVATE` | Allow webhooks to private/loopback addresses | `false` | | `OBSIGATE_PDF_MAX_SIZE_MB` | Max PDF size for text extraction | `50` | | `OBSIGATE_PDF_EXTRACT_TIMEOUT` | PDF extraction timeout (seconds) | `30` | +| `OBSIGATE_TAVILY_API_KEY` / `OBSIGATE_BRAVE_API_KEY` / `OBSIGATE_SERPAPI_API_KEY` / `OBSIGATE_EXA_API_KEY` | Keyed web-search providers (tried before SearXNG) | — | +| `OBSIGATE_WEB_PROVIDERS` | Search provider order (e.g. `brave,searxng`) | — | +| `OBSIGATE_WEB_RETRY` | Web tools network retries (house-made backoff) | `1` | +| `OBSIGATE_WEB_CACHE_TTL` | SQLite cache TTL for web results (seconds, `0` = off) | `900` | +| `OBSIGATE_GITEA_URL` / `OBSIGATE_GITEA_TOKEN` | Gitea connected source (`git_list_repos`…) | — | +| `OBSIGATE_GITHUB_TOKEN` | GitHub token (`git_list_repos`…) | — | >All these variables are documented in `.env.example`. @@ -1085,8 +1091,8 @@ This project is licensed under the **MIT License** - see the [LICENSE](LICENSE) ## 📝 Changelog -See [CHANGELOG.md](./CHANGELOG.md) for the complete version history (v1.0.0 → v2.9.1). +See [CHANGELOG.md](./CHANGELOG.md) for the complete version history (v1.0.0 → v2.10.0). --- -*Project: ObsiGate | Version: 2.9.1 | Last updated: May 2026* +*Project: ObsiGate | Version: 2.10.0 | Last updated: May 2026* diff --git a/VERSION b/VERSION index dedcc7d..10c2c0c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.9.1 +2.10.0 diff --git a/backend/requirements.txt b/backend/requirements.txt index 87d8e48..96220cb 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -20,3 +20,6 @@ psutil>=5.9 pywebpush>=2.3.0 mcp==1.9.4 sse-starlette==2.1.3 +openpyxl>=3.1 +python-docx>=1.1 +reportlab>=4.0 diff --git a/backend/services/mutations.py b/backend/services/mutations.py index ec08eaa..425ffb2 100644 --- a/backend/services/mutations.py +++ b/backend/services/mutations.py @@ -44,7 +44,7 @@ def _ensure_writable(root: Path) -> None: raise ServiceError("Vault is read-only", code="read_only", status=403) -def _validate_extension(file_path: Path, *, allow_images: bool = False) -> None: +def _validate_extension(file_path: Path, *, allow_images: bool = False, allow_docs: bool = False) -> None: """Reject unsupported file extensions (400).""" from backend.indexer import SUPPORTED_EXTENSIONS @@ -53,6 +53,9 @@ def _validate_extension(file_path: Path, *, allow_images: bool = False) -> None: if allow_images: from backend.attachment_indexer import IMAGE_EXTENSIONS allowed = allowed | IMAGE_EXTENSIONS + if allow_docs: + # Office documents produced by the AI tool layer (#92). + allowed = allowed | {".xlsx", ".docx"} if ext not in allowed and file_path.name.lower() not in ("dockerfile", "makefile"): raise ServiceError( @@ -715,17 +718,20 @@ def save_raw_file( content: bytes, *, overwrite: bool = True, + allow_docs: bool = False, ) -> dict[str, Any]: """Save a binary or text file to a vault (e.g. from upload / drag-and-drop). Creates parent directories automatically and safely validates the path. - Supports supported text extensions, images and Excalidraw files. + Supports supported text extensions, images, Excalidraw files and — with + ``allow_docs`` — Office documents (.xlsx/.docx) produced by the AI tools. Args: vault_name: Name of the vault. path: Vault-relative path. content: Raw bytes to write. overwrite: When True, replace existing files (with backup). + allow_docs: Also accept .xlsx/.docx extensions (AI document tools). Returns: Dict with ``success``, ``vault``, ``path``, and ``size``. @@ -733,7 +739,7 @@ def save_raw_file( root = get_vault_root(vault_name) _ensure_writable(root) file_path = resolve_safe_path(root, path) - _validate_extension(file_path, allow_images=True) + _validate_extension(file_path, allow_images=True, allow_docs=allow_docs) rel_path = _rel(root, file_path) diff --git a/backend/tools/api.py b/backend/tools/api.py index 3944572..0471394 100644 --- a/backend/tools/api.py +++ b/backend/tools/api.py @@ -9,6 +9,9 @@ Note: ObsiGate uses implicit namespace packages (no tracked ``__init__.py``, which ``.gitignore`` excludes via ``_*.py``), hence this explicit facade. """ +from backend.tools import connected as _connected # noqa: F401 (registers connected-source tools) +from backend.tools import crawler as _crawler # noqa: F401 (registers the site crawler) +from backend.tools import documents as _documents # noqa: F401 (registers document tools) from backend.tools import service as _service # noqa: F401 (registers tools) from backend.tools import web as _web # noqa: F401 (registers web tools) from backend.tools.context import ( diff --git a/backend/tools/connected.py b/backend/tools/connected.py new file mode 100644 index 0000000..e52354b --- /dev/null +++ b/backend/tools/connected.py @@ -0,0 +1,239 @@ +"""Connected sources — Gitea & GitHub repositories (phase 2 #92). + +The assistant can query the source-hosting platforms the project actually +uses (ObsiGate is hosted on Gitea): repositories, issues/pull requests and +repository files. Everything is READ-risk, rate-limited through the shared +registry and audited. + +Configuration (environment — injected by Infisical in production, never +hard-coded): + +* ``OBSIGATE_GITEA_URL`` — base URL of the self-hosted instance (e.g. + ``https://git.example.net``); the ``gitea`` provider is only available when + this variable is set. Admin-controlled, so the SSRF guard does not apply + (unlike user-supplied URLs). +* ``OBSIGATE_GITEA_TOKEN`` — optional personal access token (private repos). +* ``OBSIGATE_GITHUB_TOKEN`` — optional token (raises the API rate limits and + unlocks private repositories). + +Cloud drives (Google Drive / OneDrive) deliberately stay out of the core: +per the documented roadmap they are best served by an *external MCP server* +(#79) so the OAuth surface remains outside ObsiGate. +""" + +from __future__ import annotations + +import base64 +import binascii +import logging +import os +from typing import Any + +import httpx + +from backend.tools.context import ToolError, ToolRisk +from backend.tools.registry import tool +from backend.tools.schemas import GitGetFileInput, GitProviderInput, GitSearchIssuesInput + +logger = logging.getLogger("obsigate.tools.connected") + +TIMEOUT = 10.0 +USER_AGENT = "ObsiGateAssistant/1.0 (+self-hosted vault AI)" +MAX_FILE_BYTES = 300_000 +GITHUB_API = "https://api.github.com" + + +def _provider_base(provider: str) -> tuple[str, str]: + """Return (base_url, auth_header_value) for the requested provider.""" + if provider == "gitea": + base = os.environ.get("OBSIGATE_GITEA_URL", "").strip().rstrip("/") + if not base: + raise ToolError( + "Source Gitea non configurée (OBSIGATE_GITEA_URL absente).", + code="provider_not_configured", + ) + token = os.environ.get("OBSIGATE_GITEA_TOKEN", "").strip() + return base, f"token {token}" if token else "" + if provider == "github": + token = os.environ.get("OBSIGATE_GITHUB_TOKEN", "").strip() + return GITHUB_API, f"Bearer {token}" if token else "" + raise ToolError( + f"Fournisseur inconnu : {provider} ('gitea' ou 'github')", + code="invalid_arguments", + ) + + +def _headers(auth: str) -> dict[str, str]: + headers = {"User-Agent": USER_AGENT, "Accept": "application/json"} + if auth: + headers["Authorization"] = auth + return headers + + +def _request(method: str, url: str, auth: str, **kwargs: Any) -> httpx.Response: + try: + resp = httpx.request( + method, url, headers=_headers(auth), timeout=TIMEOUT, follow_redirects=False, + **kwargs, + ) + except httpx.HTTPError as e: + logger.warning("connected source request failed %s: %s", url, e) + raise ToolError( + "Source connectée momentanément indisponible.", + code="connected_source_unavailable", + ) from e + if resp.status_code in (401, 403): + raise ToolError( + "Accès refusé par la source connectée (jeton manquant ou expiré).", + code="permission_denied", + ) + if resp.status_code == 404: + raise ToolError("Ressource introuvable sur la source connectée.", code="not_found") + resp.raise_for_status() + return resp + + +def _normalize_repo(item: dict[str, Any]) -> dict[str, Any]: + return { + "name": item.get("name") or "", + "full_name": item.get("full_name") or "", + "url": item.get("html_url") or item.get("clone_url") or "", + "description": item.get("description") or "", + "updated": item.get("updated_at") or "", + "private": bool(item.get("private", False)), + } + + +@tool( + name="git_list_repos", + description=( + "List repositories on the connected Gitea instance or GitHub account " + "(name, url, description, last update). Use when the user asks about " + "their code projects." + ), + input_model=GitProviderInput, + risk=ToolRisk.READ, +) +def git_list_repos(ctx, params: GitProviderInput) -> dict[str, Any]: + """Query the configured source and return normalized repositories.""" + base, auth = _provider_base(params.provider) + if params.provider == "gitea": + url = base + "/api/v1/repos/search" + query: dict[str, Any] = {"limit": params.limit} + if params.repo: + query["q"] = params.repo + resp = _request("GET", url, auth, params=query) + items = resp.json().get("data") or [] + else: + if params.repo: + url = GITHUB_API + f"/repos/{params.repo.strip('/')}" + items = [_request("GET", url, auth).json()] + else: + resp = _request( + "GET", GITHUB_API + "/user/repos", + auth, params={"per_page": params.limit, "sort": "updated"}, + ) + items = resp.json() + repos = [_normalize_repo(item) for item in items if isinstance(item, dict)] + return {"provider": params.provider, "count": len(repos), "repos": repos} + + +@tool( + name="git_search_issues", + description=( + "Search issues and pull requests on the connected Gitea instance or " + "GitHub (title/body keywords, optional repository scope, open/closed)." + ), + input_model=GitSearchIssuesInput, + risk=ToolRisk.READ, +) +def git_search_issues(ctx, params: GitSearchIssuesInput) -> dict[str, Any]: + """Query issues (and PRs) from the configured source.""" + base, auth = _provider_base(params.provider) + state = params.state if params.state in ("open", "closed") else "open" + if params.provider == "gitea": + if params.repo: + url = base + f"/api/v1/repos/{params.repo.strip('/')}/issues" + query: dict[str, Any] = {"state": state, "limit": params.limit, "q": params.query} + resp = _request("GET", url, auth, params=query) + items = resp.json() + else: + url = base + "/api/v1/repos/issues/search" + resp = _request("GET", url, auth, params={ + "q": params.query, "state": state, "limit": params.limit, + }) + items = resp.json() + else: + clause = f"{params.query} is:issue is:{state}" + if params.repo: + clause += f" repo:{params.repo.strip('/')}" + resp = _request( + "GET", GITHUB_API + "/search/issues", auth, + params={"q": clause, "per_page": params.limit}, + ) + items = (resp.json().get("items") or []) + issues = [ + { + "id": item.get("number") or item.get("id") or "", + "title": (item.get("title") or "")[:300], + "url": item.get("html_url") or "", + "state": item.get("state") or "", + "pull_request": bool(item.get("pull_request")), + } + for item in (items if isinstance(items, list) else []) + if isinstance(item, dict) + ] + return { + "provider": params.provider, + "query": params.query, + "count": len(issues), + "issues": issues, + } + + +@tool( + name="git_get_file", + description=( + "Read a file's content from a connected Gitea or GitHub repository " + "(source code, docs, config). Text/JSON only, size-capped." + ), + input_model=GitGetFileInput, + risk=ToolRisk.READ, +) +def git_get_file(ctx, params: GitGetFileInput) -> dict[str, Any]: + """Fetch one repository file and return its decoded text content.""" + base, auth = _provider_base(params.provider) + repo = params.repo.strip("/") + path = params.path.strip("/") + if not repo or not path: + raise ToolError( + "'repo' (owner/nom) et 'path' sont obligatoires", code="invalid_arguments" + ) + if params.provider == "gitea": + url = base + f"/api/v1/repos/{repo}/contents/{path}" + else: + url = GITHUB_API + f"/repos/{repo}/contents/{path}" + if params.ref: + url += f"?ref={params.ref}" + resp = _request("GET", url, auth) + data = resp.json() + encoded = data.get("content") or "" + if (data.get("encoding") or "") == "base64" and encoded: + try: + content = base64.b64decode(encoded).decode("utf-8", errors="replace") + except (ValueError, binascii.Error) as e: + raise ToolError( + "Contenu du fichier illisible (encodage inattendu).", + code="file_decode_error", + ) from e + else: + content = encoded + truncated = len(content) > MAX_FILE_BYTES + return { + "provider": params.provider, + "repo": repo, + "path": data.get("path") or path, + "size": data.get("size") or len(content), + "content": content[:MAX_FILE_BYTES], + "truncated": truncated, + } diff --git a/backend/tools/crawler.py b/backend/tools/crawler.py new file mode 100644 index 0000000..b9952a1 --- /dev/null +++ b/backend/tools/crawler.py @@ -0,0 +1,197 @@ +"""Multi-page site crawl — ``crawl_site`` (phase 2 #92, WRITE + confirmation). + +The assistant can digest a small public site (documentation, docs portal) and +store a Markdown summary inside a vault: one section per page, title, URL and +readable text. The crawl is bounded and same-host only: + +* max 20 pages (``max_pages``), same hostname, breadth-first from the entry URL; +* SSRF guard on every URL (scheme + private-address rejection), size caps; +* no third-party crawler dependency (scrapy deliberately avoided — a bounded + httpx BFS keeps the surface small and the runtime predictable; the task is + executed as a single background-style tool run instead of a web request + pipeline). + +Risk is WRITE: the digest is written into a vault, so the two-step +confirmation applies (Apply card in the UI, propose/apply over MCP). +""" + +from __future__ import annotations + +import logging +import re +import time +from typing import Any +from urllib.parse import urljoin, urlparse + +import httpx + +from backend.tools.context import ToolContext, ToolError, ToolRisk +from backend.tools.registry import tool +from backend.tools.schemas import CrawlSiteInput +from backend.tools.web import ( + USER_AGENT, + _assert_public_http_url, + _html_to_text, + _response_text, +) + +logger = logging.getLogger("obsigate.tools.crawler") + +MAX_PAGE_BYTES = 800_000 +MAX_TOTAL_BYTES = 6_000_000 +MAX_TEXT_PER_PAGE = 12_000 +PAGE_TIMEOUT = 10.0 +_LINK_RE = re.compile(r']*href="([^"#]+)"', re.IGNORECASE) +_TITLE_RE = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) + + +def _same_host(url: str, host: str) -> bool: + return (urlparse(url).hostname or "") == host + + +def _extract_links(raw: str, base_url: str) -> list[str]: + import html as html_lib + + links: list[str] = [] + for match in _LINK_RE.finditer(raw): + href = html_lib.unescape(match.group(1)).strip() + if not href or href.lower().startswith(("javascript:", "mailto:", "tel:")): + continue + absolute = urljoin(base_url, href) + if absolute.lower().endswith((".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".pdf", ".zip")): + continue + links.append(absolute.split("#", 1)[0]) + return links + + +def _fetch_page(url: str) -> tuple[str, str]: + """Fetch one page (SSRF-guarded, manual redirects) → (title, text).""" + current = _assert_public_http_url(url) + resp = None + for _hop in range(5): + resp = httpx.get( + current, + headers={"User-Agent": USER_AGENT, "Accept": "text/html,*/*"}, + timeout=PAGE_TIMEOUT, + follow_redirects=False, + ) + if resp.status_code in (301, 302, 303, 307, 308): + location = resp.headers.get("location") or "" + if not location: + break + current = _assert_public_http_url(str(httpx.URL(current).join(location))) + continue + break + assert resp is not None + resp.raise_for_status() + ctype = (resp.headers.get("content-type") or "").lower() + if "html" not in ctype and "text" not in ctype: + raise ToolError( + f"Type de contenu non pris en charge: {ctype.split(';')[0] or 'inconnu'}", + code="unsupported_content_type", + ) + raw = (resp.content[:MAX_PAGE_BYTES]).decode(resp.encoding or "utf-8", errors="replace") + title_match = _TITLE_RE.search(raw) + import html as html_lib + + title = html_lib.unescape(title_match.group(1)).strip()[:300] if title_match else "" + return title, _html_to_text(raw)[:MAX_TEXT_PER_PAGE] + + +@tool( + name="crawl_site", + description=( + "Crawl a small public site (same-host only, max 20 pages) starting at " + "a URL and save a Markdown digest (title, url, readable text per page) " + "into a vault. Use to capture an online documentation for offline use." + ), + input_model=CrawlSiteInput, + risk=ToolRisk.WRITE, + requires_vault=True, +) +def crawl_site(ctx: ToolContext, params: CrawlSiteInput) -> dict[str, Any]: + """Bounded BFS crawl; writes the digest file and returns a summary.""" + from backend.services.errors import ServiceError + from backend.services.mutations import save_raw_file + + start = _assert_public_http_url(params.url.strip()) + host = urlparse(start).hostname or "" + if not host: + raise ToolError("URL sans hôte", code="invalid_url") + + queue: list[str] = [start] + seen: set[str] = {start} + pages: list[dict[str, Any]] = [] + total_bytes = 0 + failures: list[str] = [] + + while queue and len(pages) < params.max_pages and total_bytes < MAX_TOTAL_BYTES: + url = queue.pop(0) + try: + title, text = _fetch_page(url) + except ToolError as e: + failures.append(url) + logger.warning("crawl_site page failed %s: %s", url, e.code) + continue + except httpx.HTTPError as e: + failures.append(url) + logger.warning("crawl_site page failed %s: %s", url, e) + continue + pages.append({"url": url, "title": title, "text": text}) + total_bytes += len(text) + if len(pages) >= params.max_pages: + break + try: + raw_resp = httpx.get( + url, headers={"User-Agent": USER_AGENT}, timeout=PAGE_TIMEOUT, + follow_redirects=False, + ) + raw = _response_text(raw_resp) + except (httpx.HTTPError, ValueError): + continue + for link in _extract_links(raw, url): + if len(pages) + len(queue) >= params.max_pages: + break + if link in seen or not _same_host(link, host): + continue + try: + _assert_public_http_url(link) + except ToolError: + continue + seen.add(link) + queue.append(link) + + if not pages: + raise ToolError( + "Aucune page n'a pu être récupérée pour ce site.", + code="crawl_failed", + ) + + lines = [ + f"# Crawl de {host}", + "", + f"> {len(pages)} page(s) capturée(s) depuis {start} — {time.strftime('%Y-%m-%d %H:%M')}", + "", + ] + for page in pages: + lines.append(f"## {page['title'] or page['url']}") + lines.append("") + lines.append(f"Source : {page['url']}") + lines.append("") + lines.append(page["text"]) + lines.append("") + digest = "\n".join(lines).encode("utf-8") + try: + saved = save_raw_file( + params.vault, params.path, digest, overwrite=True, allow_docs=False + ) + except ServiceError as e: + raise ToolError(e.message, code=e.code, details=e.details) from e + return { + "url": start, + "vault": params.vault, + "path": saved.get("path", params.path), + "pages": len(pages), + "failed": failures[:10], + "size": saved.get("size", len(digest)), + } diff --git a/backend/tools/documents.py b/backend/tools/documents.py new file mode 100644 index 0000000..ac48d21 --- /dev/null +++ b/backend/tools/documents.py @@ -0,0 +1,188 @@ +"""Document-production tools (phase 2 #92) — WRITE, confirmation required. + +The assistant can generate real files inside a vault: + +* ``create_xlsx`` — spreadsheet (openpyxl); +* ``create_docx`` — Word document (python-docx); +* ``create_csv`` — CSV (stdlib); +* ``create_pdf`` — PDF (reportlab, from markdown-ish content). + +Every tool is ``WRITE`` (two-step confirm in the UI / propose-apply over MCP), +vault-scoped through ``requires_vault`` and saved via the shared mutation +service (path safety, read-only check, backup on overwrite). +""" + +from __future__ import annotations + +import csv as csv_lib +import io +import logging +import re +from typing import Any +from xml.sax import saxutils + +from backend.services.errors import ServiceError +from backend.services.mutations import save_raw_file +from backend.tools.context import ToolContext, ToolError, ToolRisk +from backend.tools.registry import tool +from backend.tools.schemas import CsvInput, DocxInput, PdfInput, SpreadsheetInput + +logger = logging.getLogger("obsigate.tools.documents") + +MAX_PDF_CHARS = 200_000 +MAX_ROWS = 5_000 + + +def _save(vault: str, path: str, content: bytes, overwrite: bool) -> dict[str, Any]: + """Shared save helper (maps ServiceError to ToolError).""" + try: + return save_raw_file(vault, path, content, overwrite=overwrite, allow_docs=True) + except ServiceError as e: + raise ToolError(e.message, code=e.code, details=e.details) from e + + +def _check_rows(rows: list[list[Any]]) -> None: + if not rows: + raise ToolError("Aucune ligne fournie", code="invalid_arguments") + if len(rows) > MAX_ROWS: + raise ToolError( + f"Trop de lignes ({len(rows)} > {MAX_ROWS})", code="invalid_arguments" + ) + + +def _check_extension(path: str, expected: str) -> str: + """Enforce the document extension; return the normalized path.""" + path = (path or "").strip() + if not path.lower().endswith(expected): + raise ToolError( + f"Extension attendue : {expected}", code="invalid_arguments" + ) + return path + + +@tool( + name="create_xlsx", + description=( + "Create an .xlsx spreadsheet in a vault from rows of cell values " + "(first row = header). Use for tables, budgets, checklists the user " + "asked to turn into an Excel file." + ), + input_model=SpreadsheetInput, + risk=ToolRisk.WRITE, + requires_vault=True, +) +def create_xlsx(ctx: ToolContext, params: SpreadsheetInput) -> dict[str, Any]: + """Build the workbook with openpyxl and save it into the vault.""" + from openpyxl import Workbook + + _check_rows(params.rows) + path = _check_extension(params.path, ".xlsx") + wb = Workbook() + ws = wb.active + ws.title = params.sheet_name[:31] or "Feuille1" + for row in params.rows: + ws.append(list(row)) + buffer = io.BytesIO() + wb.save(buffer) + return _save(params.vault, path, buffer.getvalue(), params.overwrite) + + +@tool( + name="create_docx", + description=( + "Create a .docx Word document in a vault from an optional title and " + "ordered paragraphs. Use for letters, reports, structured drafts." + ), + input_model=DocxInput, + risk=ToolRisk.WRITE, + requires_vault=True, +) +def create_docx(ctx: ToolContext, params: DocxInput) -> dict[str, Any]: + """Build the document with python-docx and save it into the vault.""" + from docx import Document + + if not params.paragraphs: + raise ToolError("Aucun paragraphe fourni", code="invalid_arguments") + path = _check_extension(params.path, ".docx") + doc = Document() + if params.title.strip(): + doc.add_heading(params.title.strip(), level=1) + for paragraph in params.paragraphs: + doc.add_paragraph(paragraph) + buffer = io.BytesIO() + doc.save(buffer) + return _save(params.vault, path, buffer.getvalue(), params.overwrite) + + +@tool( + name="create_csv", + description=( + "Create a .csv file in a vault from rows of cell values (first row = " + "header). Use for flat data exports, simple tables." + ), + input_model=CsvInput, + risk=ToolRisk.WRITE, + requires_vault=True, +) +def create_csv(ctx: ToolContext, params: CsvInput) -> dict[str, Any]: + """Serialize the rows and save the CSV into the vault.""" + _check_rows(params.rows) + path = _check_extension(params.path, ".csv") + delimiter = params.delimiter if params.delimiter in (",", ";", "\t") else "," + buffer = io.StringIO() + writer = csv_lib.writer(buffer, delimiter=delimiter, lineterminator="\n") + writer.writerows(params.rows) + return _save(params.vault, path, buffer.getvalue().encode("utf-8"), params.overwrite) + + +_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*)$") + + +def _markdown_to_flowables(content: str) -> list[tuple[str, str]]: + """Split markdown-ish content into (style, text) blocks for reportlab.""" + blocks: list[tuple[str, str]] = [] + for raw_line in content.splitlines(): + line = raw_line.rstrip() + if not line.strip(): + continue + heading = _HEADING_RE.match(line) + if heading: + blocks.append((f"H{min(3, len(heading.group(1)))}", heading.group(2).strip())) + else: + blocks.append(("P", line.strip())) + return blocks + + +@tool( + name="create_pdf", + description=( + "Create a .pdf document in a vault from markdown-ish content " + "(# headings + paragraphs). Use for printable deliverables." + ), + input_model=PdfInput, + risk=ToolRisk.WRITE, + requires_vault=True, +) +def create_pdf(ctx: ToolContext, params: PdfInput) -> dict[str, Any]: + """Render the content with reportlab and save the PDF into the vault.""" + from reportlab.lib.pagesizes import A4 + from reportlab.lib.styles import getSampleStyleSheet + from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer + + path = _check_extension(params.path, ".pdf") + text = params.content[:MAX_PDF_CHARS] + styles = getSampleStyleSheet() + style_map = { + "P": styles["BodyText"], + "H1": styles["Heading1"], + "H2": styles["Heading2"], + "H3": styles["Heading3"], + } + buffer = io.BytesIO() + doc = SimpleDocTemplate(buffer, pagesize=A4, title=params.title[:200]) + story: list[Any] = [Paragraph(saxutils.escape(params.title[:300]), styles["Title"])] + for style, line in _markdown_to_flowables(text): + story.append(Spacer(1, 4)) + story.append(Paragraph(saxutils.escape(line), style_map[style])) + doc.build(story) + return _save(params.vault, path, buffer.getvalue(), params.overwrite) diff --git a/backend/tools/labels.py b/backend/tools/labels.py index 15325d4..1985cb8 100644 --- a/backend/tools/labels.py +++ b/backend/tools/labels.py @@ -47,6 +47,14 @@ _STEP_LABELS: dict[str, tuple[str, str | None]] = { "restore_backup": ("backup_restore", "path"), "web_search": ("web_search", "query"), "fetch_url": ("fetch_url", "url"), + "crawl_site": ("crawl", "url"), + "git_list_repos": ("git_repos", "provider"), + "git_search_issues": ("git_issues", "query"), + "git_get_file": ("git_file", "path"), + "create_xlsx": ("xlsx_create", "path"), + "create_docx": ("docx_create", "path"), + "create_csv": ("csv_create", "path"), + "create_pdf": ("pdf_create", "path"), } GENERIC_KEY = "generic" diff --git a/backend/tools/schemas.py b/backend/tools/schemas.py index 601b93f..7a0da66 100644 --- a/backend/tools/schemas.py +++ b/backend/tools/schemas.py @@ -251,6 +251,90 @@ class FetchUrlInput(BaseModel): """Fetch one public web page and return its readable text.""" url: str = Field(..., description="Absolute http(s) URL of a public page") + render: bool = Field( + False, + description="Render JavaScript with the optional Playwright worker (dynamic SPA pages)", + ) + + +class CrawlSiteInput(BaseModel): + """Crawl a small public site (same-host only) and save a digest into a vault.""" + + url: str = Field(..., description="Absolute http(s) URL where the crawl starts") + vault: str = Field(..., description="Vault name") + path: str = Field(..., description="Vault-relative path of the digest file to write (.md)") + max_pages: int = Field(5, ge=1, le=20, description="Maximum number of pages to crawl") + + +class GitProviderInput(BaseModel): + """Base fields for connected-source tools (Gitea / GitHub).""" + + provider: str = Field(..., description="'gitea' (OBSIGATE_GITEA_URL) or 'github'") + repo: str = Field("", description="Optional 'owner/name' repository filter") + limit: int = Field(20, ge=1, le=50, description="Maximum number of entries") + + +class GitSearchIssuesInput(BaseModel): + """Search issues/pull requests on a connected Gitea or GitHub instance.""" + + provider: str = Field(..., description="'gitea' or 'github'") + query: str = Field(..., min_length=1, description="Search keywords") + repo: str = Field("", description="Optional 'owner/name' scope (empty = instance-wide)") + state: str = Field("open", description="'open' or 'closed'") + limit: int = Field(10, ge=1, le=20, description="Maximum number of issues") + + +class GitGetFileInput(BaseModel): + """Read a file from a connected Gitea or GitHub repository.""" + + provider: str = Field(..., description="'gitea' or 'github'") + repo: str = Field(..., description="'owner/name' repository") + path: str = Field(..., description="Repository-relative file path") + ref: str = Field("", description="Optional branch/tag/commit (empty = default branch)") + + +class SpreadsheetInput(BaseModel): + """Create an .xlsx spreadsheet in a vault from rows of cells.""" + + vault: str = Field(..., description="Vault name") + path: str = Field(..., description="Vault-relative path of the file to write (.xlsx)") + rows: list[list[str | int | float | bool | None]] = Field( + ..., description="Rows of cell values (first row = header)" + ) + sheet_name: str = Field("Feuille1", description="Worksheet name") + overwrite: bool = Field(True, description="Replace an existing file (with backup)") + + +class DocxInput(BaseModel): + """Create a .docx Word document in a vault from paragraphs.""" + + vault: str = Field(..., description="Vault name") + path: str = Field(..., description="Vault-relative path of the file to write (.docx)") + title: str = Field("", description="Optional document title (heading 1)") + paragraphs: list[str] = Field(..., description="Paragraph texts, in order") + overwrite: bool = Field(True, description="Replace an existing file (with backup)") + + +class CsvInput(BaseModel): + """Create a .csv file in a vault from rows of cells.""" + + vault: str = Field(..., description="Vault name") + path: str = Field(..., description="Vault-relative path of the file to write (.csv)") + rows: list[list[str | int | float | bool | None]] = Field( + ..., description="Rows of cell values (first row = header)" + ) + delimiter: str = Field(",", description="Field separator (',' ';' '\\t')") + overwrite: bool = Field(True, description="Replace an existing file (with backup)") + + +class PdfInput(BaseModel): + """Create a .pdf document in a vault from markdown-ish content.""" + + vault: str = Field(..., description="Vault name") + path: str = Field(..., description="Vault-relative path of the file to write (.pdf)") + title: str = Field("Document", description="Document title") + content: str = Field(..., description="Content (headings with #/##, then paragraphs)") + overwrite: bool = Field(True, description="Replace an existing file (with backup)") class ToolResult(BaseModel): diff --git a/backend/tools/web.py b/backend/tools/web.py index 46ec358..b62d1d9 100644 --- a/backend/tools/web.py +++ b/backend/tools/web.py @@ -8,6 +8,16 @@ Phase 1 of the documented web-toolset roadmap: answering « je n'ai pas accès à internet ». * ``fetch_url`` — retrieve a public web page and return readable text. +Phase 2 (#92) additions: + +* keyed providers — Tavily, Brave Search, SerpAPI and Exa are used first when + their API key is configured (env, injected by Infisical in production); +* SQLite cache — search/fetch results are cached with a TTL + (:mod:`backend.tools.webcache`); +* retry with backoff — transient network errors get one extra attempt; +* dynamic rendering — ``fetch_url(render=True)`` uses an isolated Playwright + worker (optional dependency, graceful degradation). + All are READ-risk tools (no confirmation), rate-limited through the shared registry, SSRF-guarded (scheme + private-address rejection), and size-capped. @@ -16,6 +26,14 @@ Configuration (environment): * ``OBSIGATE_WEB_TIMEOUT`` — seconds, default 10 * ``OBSIGATE_WEB_FALLBACK`` — ``0``/``false`` disables the keyless HTML fallbacks (SearXNG only), default enabled +* ``OBSIGATE_TAVILY_API_KEY`` / ``OBSIGATE_BRAVE_API_KEY`` / + ``OBSIGATE_SERPAPI_API_KEY`` / ``OBSIGATE_EXA_API_KEY`` — optional keyed + providers, tried before SearXNG when set +* ``OBSIGATE_WEB_PROVIDERS`` — optional comma-separated provider order + (e.g. ``brave,searxng``); keyed providers without a key are skipped +* ``OBSIGATE_WEB_RETRY`` — extra attempts for transient network errors + (default 1) +* ``OBSIGATE_WEB_CACHE_TTL`` — cache TTL seconds, ``0`` disables (default 900) """ from __future__ import annotations @@ -28,12 +46,14 @@ import logging import os import re import socket +import time from collections.abc import Callable from typing import Any from urllib.parse import parse_qs, urlparse import httpx +from backend.tools import webcache from backend.tools.context import ToolError, ToolRisk, ToolScope from backend.tools.registry import tool from backend.tools.schemas import FetchUrlInput, WebSearchInput @@ -48,6 +68,7 @@ WEB_FALLBACK_ENABLED = os.environ.get("OBSIGATE_WEB_FALLBACK", "1").strip().lowe "no", "off", } +WEB_RETRY_ATTEMPTS = int(os.environ.get("OBSIGATE_WEB_RETRY", "1")) USER_AGENT = "ObsiGateAssistant/1.0 (+self-hosted vault AI)" # Search engines reject non-browser agents on their public HTML endpoints. BROWSER_UA = ( @@ -163,6 +184,115 @@ def _result( } +def _with_retry(call: Callable[[], Any]) -> Any: + """Run *call* with one extra attempt on transient network errors. + + House-made backoff (the roadmap's « tenacity ou boucle maison »): DNS + blips and rate-limit hiccups are the common failure mode, and a single + retry keeps the fallback chain from being consumed too early. + """ + for attempt in range(1 + max(0, WEB_RETRY_ATTEMPTS)): + try: + return call() + except httpx.TransportError: + if attempt >= max(0, WEB_RETRY_ATTEMPTS): + raise + time.sleep(0.2 * (attempt + 1)) + raise RuntimeError("unreachable") # pragma: no cover + + +def _env_key(name: str) -> str: + """Read an API key from the environment (Infisical-injected in prod).""" + return os.environ.get(name, "").strip() + + +def _search_tavily(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]: + """Tavily Search API (agent-oriented results, key required).""" + resp = httpx.post( + "https://api.tavily.com/search", + json={ + "api_key": _env_key("OBSIGATE_TAVILY_API_KEY"), + "query": query, + "max_results": params.max_results, + "search_depth": "basic", + "include_answer": False, + }, + headers={"User-Agent": USER_AGENT}, + timeout=WEB_TIMEOUT, + ) + resp.raise_for_status() + data = resp.json() + return [ + _result(item.get("title") or "", item.get("url") or "", item.get("content") or "") + for item in (data.get("results") or []) + ], [] + + +def _search_brave(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]: + """Brave Search API (key required).""" + resp = httpx.get( + "https://api.search.brave.com/res/v1/web/search", + params={"q": query, "count": params.max_results, "safesearch": "moderate"}, + headers={ + "X-Subscription-Id": _env_key("OBSIGATE_BRAVE_API_KEY"), + "Accept": "application/json", + "User-Agent": USER_AGENT, + }, + timeout=WEB_TIMEOUT, + ) + resp.raise_for_status() + data = resp.json() + return [ + _result(item.get("title") or "", item.get("url") or "", item.get("description") or "") + for item in ((data.get("web") or {}).get("results") or []) + ], [] + + +def _search_serpapi(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]: + """SerpAPI (Google SERP, key required).""" + resp = httpx.get( + "https://serpapi.com/search", + params={"q": query, "api_key": _env_key("OBSIGATE_SERPAPI_API_KEY"), + "num": params.max_results}, + headers={"User-Agent": USER_AGENT}, + timeout=WEB_TIMEOUT, + ) + resp.raise_for_status() + data = resp.json() + return [ + _result(item.get("title") or "", item.get("link") or "", item.get("snippet") or "") + for item in (data.get("organic_results") or []) + ], [] + + +def _search_exa(query: str, params: WebSearchInput) -> tuple[list[dict[str, Any]], list[str]]: + """Exa neural search (key required).""" + resp = httpx.post( + "https://api.exa.ai/search", + json={"query": query, "numResults": params.max_results}, + headers={ + "x-api-key": _env_key("OBSIGATE_EXA_API_KEY"), + "User-Agent": USER_AGENT, + }, + timeout=WEB_TIMEOUT, + ) + resp.raise_for_status() + data = resp.json() + return [ + _result(item.get("title") or "", item.get("url") or "", (item.get("text") or "")[:600]) + for item in (data.get("results") or []) + ], [] + + +# Keyed providers: name -> (implementation, API key env var) +_KEYED_PROVIDERS: dict[str, tuple[_Provider, str]] = { + "tavily": (_search_tavily, "OBSIGATE_TAVILY_API_KEY"), + "brave": (_search_brave, "OBSIGATE_BRAVE_API_KEY"), + "serpapi": (_search_serpapi, "OBSIGATE_SERPAPI_API_KEY"), + "exa": (_search_exa, "OBSIGATE_EXA_API_KEY"), +} + + def _search_searxng( query: str, params: WebSearchInput ) -> tuple[list[dict[str, Any]], list[str]]: @@ -288,8 +418,22 @@ _Provider = Callable[[str, WebSearchInput], "tuple[list[dict[str, Any]], list[st def _provider_chain() -> list[tuple[str, _Provider]]: - """Ordered providers: self-hosted meta-search first, then keyless fallbacks.""" - chain: list[tuple[str, _Provider]] = [("searxng", _search_searxng)] + """Ordered providers: keyed APIs first, then self-hosted, then keyless. + + ``OBSIGATE_WEB_PROVIDERS`` (comma-separated) overrides the default order; + unknown names are ignored and keyed providers without their key are skipped. + """ + chain: list[tuple[str, _Provider]] = [] + configured = [ + name.strip().lower() + for name in os.environ.get("OBSIGATE_WEB_PROVIDERS", "").split(",") + if name.strip() + ] + for name in configured or list(_KEYED_PROVIDERS): + entry = _KEYED_PROVIDERS.get(name) + if entry and _env_key(entry[1]): + chain.append((name, entry[0])) + chain.append(("searxng", _search_searxng)) if WEB_FALLBACK_ENABLED: chain.append(("duckduckgo", _search_duckduckgo)) chain.append(("bing", _search_bing)) @@ -313,6 +457,17 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]: if not query: raise ToolError("Requête vide", code="invalid_arguments") + key = webcache.cache_key("search", { + "q": query, + "max_results": params.max_results, + "category": params.category, + "language": params.language, + "page": params.page, + }) + cached = webcache.cache_get(key) + if cached is not None: + return {**cached, "cached": True} + attempts: list[str] = [] unresponsive: list[str] = [] reachable = False @@ -320,8 +475,12 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]: for name, provider in _provider_chain(): attempts.append(name) + + def _attempt(p: _Provider = provider) -> tuple[list[dict[str, Any]], list[str]]: + return p(query, params) + try: - results, engines = provider(query, params) + results, engines = _with_retry(_attempt) except (httpx.HTTPError, ValueError, AttributeError) as e: logger.warning("web_search provider %s failed: %s", name, e) last_error = e @@ -338,6 +497,7 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]: } if unresponsive: payload["unresponsive_engines"] = unresponsive[:8] + webcache.cache_set(key, payload) return payload if not reachable: @@ -378,6 +538,20 @@ def web_search(ctx, params: WebSearchInput) -> dict[str, Any]: def fetch_url(ctx, params: FetchUrlInput) -> dict[str, Any]: """Retrieve one page, guard against SSRF, and extract its text.""" url = _assert_public_http_url(params.url.strip()) + key = webcache.cache_key("fetch", {"url": url, "render": params.render}) + cached = webcache.cache_get(key) + if cached is not None: + return {**cached, "cached": True} + + if params.render: + # Dynamic pages (SPA/React): delegated to the isolated Playwright + # worker; the browser dependency stays optional (graceful error). + from backend.tools.webrender import render_page + + payload = render_page(url) + webcache.cache_set(key, payload) + return payload + try: # Follow redirects manually so every hop is re-checked against the # private-address SSRF guard (a public page can redirect to 127.0.0.1). @@ -414,10 +588,12 @@ def fetch_url(ctx, params: FetchUrlInput) -> dict[str, Any]: title_match = re.search(r"]*>(.*?)", raw, re.IGNORECASE | re.DOTALL) title = html_lib.unescape(title_match.group(1)).strip()[:300] if title_match else "" text = _html_to_text(raw)[:MAX_TEXT_CHARS] - return { + payload = { "url": str(resp.url), "status": resp.status_code, "title": title, "text": text, "truncated": len(raw) > MAX_TEXT_CHARS, } + webcache.cache_set(key, payload) + return payload diff --git a/backend/tools/webcache.py b/backend/tools/webcache.py new file mode 100644 index 0000000..25564bd --- /dev/null +++ b/backend/tools/webcache.py @@ -0,0 +1,138 @@ +"""SQLite cache for web tool results (search results, fetched pages). + +Phase 2 of the web-toolset roadmap (« Transverse »): repeated web searches and +page fetches (common in agent loops, where the model re-reads a source) must +not hammer the providers. Results are cached in a dedicated SQLite table with +a TTL; the cache is best-effort — any error silently disables it so a broken +database file never takes the assistant down. + +Configuration (environment): +* ``OBSIGATE_DATA_DIR`` — base data directory (default ``data``) +* ``OBSIGATE_WEB_CACHE_PATH`` — explicit cache file override +* ``OBSIGATE_WEB_CACHE_TTL`` — seconds, ``0`` disables the cache (default 900) +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import sqlite3 +import threading +import time +from pathlib import Path +from typing import Any + +logger = logging.getLogger("obsigate.tools.webcache") + +DEFAULT_TTL_SECONDS = 900 +_schema_ready = False +_write_lock = threading.Lock() + + +def ttl_seconds() -> float: + """Configured TTL in seconds (``0`` = cache disabled).""" + return float(os.environ.get("OBSIGATE_WEB_CACHE_TTL", str(DEFAULT_TTL_SECONDS))) + + +def _cache_path() -> Path: + override = os.environ.get("OBSIGATE_WEB_CACHE_PATH", "").strip() + if override: + return Path(override) + return Path(os.environ.get("OBSIGATE_DATA_DIR", "data")) / "web_cache.sqlite3" + + +def _connect() -> sqlite3.Connection: + """Open (and lazily create) the cache database.""" + global _schema_ready + path = _cache_path() + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path, timeout=5, check_same_thread=False) + if not _schema_ready: + conn.execute( + "CREATE TABLE IF NOT EXISTS web_cache (" + "key TEXT PRIMARY KEY, value TEXT NOT NULL, created REAL NOT NULL)" + ) + conn.commit() + _schema_ready = True + return conn + + +def cache_key(prefix: str, payload: dict[str, Any]) -> str: + """Deterministic cache key from a prefix and the normalized arguments.""" + raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str) + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32] + return f"{prefix}:{digest}" + + +def cache_get(key: str) -> Any | None: + """Return the cached payload for *key*, or ``None`` (miss/expiry/disabled).""" + if ttl_seconds() <= 0: + return None + try: + conn = _connect() + row = conn.execute( + "SELECT value, created FROM web_cache WHERE key = ?", (key,) + ).fetchone() + conn.close() + except sqlite3.Error as e: + logger.warning("web cache read failed (%s): %s", key, e) + return None + if row is None: + return None + value, created = row + if time.time() - float(created) > ttl_seconds(): + return None + try: + return json.loads(value) + except (ValueError, TypeError): + return None + + +def cache_set(key: str, value: Any) -> None: + """Store *value* under *key* (best effort, never raises).""" + if ttl_seconds() <= 0: + return + try: + with _write_lock: + conn = _connect() + conn.execute( + "INSERT INTO web_cache (key, value, created) VALUES (?, ?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value, created = excluded.created", + (key, json.dumps(value, ensure_ascii=False, default=str), time.time()), + ) + conn.commit() + conn.close() + except sqlite3.Error as e: + logger.warning("web cache write failed (%s): %s", key, e) + + +def purge_expired() -> int: + """Delete expired rows; return the number of removed entries (maintenance).""" + try: + conn = _connect() + cursor = conn.execute( + "DELETE FROM web_cache WHERE created < ?", (time.time() - ttl_seconds(),) + ) + conn.commit() + deleted = cursor.rowcount + conn.close() + return int(deleted) + except sqlite3.Error as e: + logger.warning("web cache purge failed: %s", e) + return 0 + + +def clear_cache() -> int: + """Drop every cached entry (tests / admin); returns the number of rows.""" + try: + conn = _connect() + cursor = conn.execute("DELETE FROM web_cache") + conn.commit() + deleted = cursor.rowcount + conn.close() + return int(deleted) + except sqlite3.Error as e: + logger.warning("web cache clear failed: %s", e) + return 0 diff --git a/backend/tools/webrender.py b/backend/tools/webrender.py new file mode 100644 index 0000000..900e00e --- /dev/null +++ b/backend/tools/webrender.py @@ -0,0 +1,100 @@ +"""Dynamic page rendering (Playwright) — ``fetch_url(render=True)``. + +Static pages are fetched with httpx inside :mod:`backend.tools.web`. Dynamic +pages (SPA/React, JS-loaded content) need a real browser engine; this module +runs one Playwright call inside a dedicated worker thread so browser +crashes/timeouts never take over the tool layer, and the heavyweight +dependency stays optional: + +* not installed → ``ToolError(code="playwright_unavailable")`` with a clear + message (the assistant explains the limitation instead of hanging); +* installed → ``pip install playwright && playwright install chromium``. + +The SSRF guard (scheme + private-address rejection) is applied before the +browser navigates. Note: unlike the httpx path, internal redirects performed +by the browser engine are not re-checked hop by hop. +""" + +from __future__ import annotations + +import html as html_lib +import logging +import re +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +from backend.tools.context import ToolError +from backend.tools.web import ( + MAX_TEXT_CHARS, + USER_AGENT, + _assert_public_http_url, + _html_to_text, +) + +logger = logging.getLogger("obsigate.tools.webrender") + +# One worker: browser automation is serialized on purpose (one Chromium at a +# time keeps memory predictable on small hosts). +_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="obsigate-playwright") +GOTO_TIMEOUT_MS = 20_000 + + +def _playwright_available() -> bool: + try: + import playwright # noqa: F401 + except ImportError: + return False + return True + + +def _render_in_worker(url: str) -> dict[str, Any]: + """Synchronous Playwright render — runs in the dedicated worker thread.""" + from playwright.sync_api import sync_playwright + + status = 0 + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + try: + page = browser.new_page(user_agent=USER_AGENT) + response = page.goto(url, wait_until="networkidle", timeout=GOTO_TIMEOUT_MS) + if response is not None: + status = response.status + raw = page.content() + title = html_lib.unescape(page.title() or "").strip() + text = _html_to_text(raw)[:MAX_TEXT_CHARS] + finally: + browser.close() + title = re.sub(r"\s+", " ", title)[:300] + return { + "url": url, + "status": status, + "title": title, + "text": text, + "rendered": True, + "truncated": len(raw) > MAX_TEXT_CHARS, + } + + +def render_page(url: str) -> dict[str, Any]: + """Render *url* (JavaScript included) and return readable text. + + Raises: + ToolError: ``playwright_unavailable`` when the optional dependency is + missing, ``render_unavailable`` when the render itself failed. + """ + _assert_public_http_url(url) + if not _playwright_available(): + raise ToolError( + "Rendu dynamique indisponible : Playwright n'est pas installé " + "(pip install playwright && playwright install chromium).", + code="playwright_unavailable", + ) + try: + return _executor.submit(_render_in_worker, url).result(timeout=GOTO_TIMEOUT_MS / 1000 + 40) + except ToolError: + raise + except Exception as e: + logger.warning("render_page failed for %s: %s", url, e) + raise ToolError( + "Le rendu dynamique de la page a échoué.", code="render_unavailable" + ) from e diff --git a/desktop/Cargo.lock b/desktop/Cargo.lock index fbc0a7e..ffd0a86 100644 --- a/desktop/Cargo.lock +++ b/desktop/Cargo.lock @@ -2626,7 +2626,7 @@ dependencies = [ [[package]] name = "obsigate-desktop" -version = "2.9.1" +version = "2.10.0" dependencies = [ "chrono", "env_logger", diff --git a/desktop/Cargo.toml b/desktop/Cargo.toml index 21beb48..77ff7a3 100644 --- a/desktop/Cargo.toml +++ b/desktop/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "obsigate-desktop" -version = "2.9.1" +version = "2.10.0" description = "ObsiGate Desktop — Porte d'entrée native pour vos vaults Obsidian" authors = ["Bruno Charest"] edition = "2021" diff --git a/desktop/tauri.conf.json b/desktop/tauri.conf.json index adfac19..8369c90 100644 --- a/desktop/tauri.conf.json +++ b/desktop/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://raw.githubusercontent.com/nicedoc/obsigate/main/desktop/tauri.conf.schema.json", "productName": "ObsiGate", - "version": "2.9.1", + "version": "2.10.0", "identifier": "com.obsigate.desktop", "build": { "frontendDist": "../frontend", diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index f65f1cb..f562fe5 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,6 +1,6 @@ # ObsiGate — Roadmap -> **Version :** 2.9.1 | **Dernière mise à jour :** 2026-09-17 +> **Version :** 2.10.0 | **Dernière mise à jour :** 2026-09-17 > **Ce fichier ne contient que le travail à venir** (🔵 En cours + ⚪ Backlog) et un index compact > vers les fonctionnalités livrées. > - **Méthode de livraison à appliquer pour toute tâche : [DELIVERY_WORKFLOW.md](./DELIVERY_WORKFLOW.md)** @@ -78,23 +78,6 @@ - [x] Personnalisation (clé à molette) : ajouter / supprimer / réordonner les commandes - [x] i18n FR/EN + tests frontend (helpers purs) + E2E mobile -### 92. Assistant IA — Écosystème d'outils (phase 2 : web étendu, sources connectées, documents) - -- **Effort :** 3-5 jours | **Impact :** 🟠 | **Zone :** backend (`backend/tools/`) -- **Dépend de :** #91 (registre + section « steps » + `web_search`/`fetch_url` livrés) -- **Description :** étendre le catalogue d'outils de l'assistant au-delà du vault, en - suivant la feuille de route technique détaillée : - [features/ai-tools-roadmap.md](./features/ai-tools-roadmap.md) (frameworks évalués, - bibliothèques par catégorie, transverse retry/cache/secrets/async). -- **Sous-tâches :** - - [x] `web_search` : chaîne de repli sans clé (SearXNG → DuckDuckGo → Bing, `OBSIGATE_WEB_FALLBACK`) — BUG-051 - - [ ] `web_search` : fournisseurs optionnels à clé (Tavily, Brave, SerpAPI, Exa) - - [ ] `fetch_url` : pages dynamiques via Playwright (worker isolé) ; crawl multi-pages Scrapy en tâche de fond - - [ ] Sources connectées : Gitea/GitHub (priorité haute) puis Google Drive / OneDrive (OAuth2 `authlib`) - - [ ] Production de documents : conversion, tableurs, PDF/Word (outils WRITE + confirmation) - - [ ] Transverse : `tenacity` (backoff), cache SQLite des résultats web avec TTL, secrets via Infisical - - [ ] Chaque outil : libellé `labels.py` + clés i18n `ai.step.*` FR/EN + tests (httpx mocké) - --- ## ⚪ Backlog — Sécurité, architecture & performance (P0/P1) @@ -201,6 +184,7 @@ | 101 | Forge — Assistant IA partagé (bouton AI Panel = assistant, fournisseur/modèle configuré, autocomplétion) + plein écran Forge/Editer | 2.8.0 | [features/forge-assistant.md](./features/forge-assistant.md) | | BUG-057 | Assistant IA — bouton « Ajouter » fonctionnel dans l'éditeur Forge (en plus d'« Editer ») | 2.9.0 | [archive](./archive/COMPLETED_v1-v2.md) | | 102 | Assistant IA — bouton « Ajouter la section » par bloc de code (insertion du bloc seul) | 2.9.0 | [archive](./archive/COMPLETED_v1-v2.md) | +| 92 | Assistant IA — Écosystème d'outils phase 2 (recherche à clé, cache/retry, Playwright, crawl, Gitea/GitHub, documents XLSX/DOCX/CSV/PDF) | 2.10.0 | [features/ai-tools-roadmap.md](./features/ai-tools-roadmap.md) | --- @@ -208,12 +192,11 @@ | Priorité | Items | Effort total estimé | |---|---|---| -| ✅ Complété | #1 → #59, #61–72, #74–76, #78–84, #88–93, #94–100 | ~114 jours réalisés | +| ✅ Complété | #1 → #59, #61–72, #74–76, #78–84, #88–93, #94–100, #102, #92 | ~114 jours réalisés | | 🔵 P2 restant | #77 Desktop : signature de code (non retenue), 6 tests E2E **manuels** ([protocole](./DESKTOP_E2E_CHECKLIST.md)) | ~0,5-1 jour | | ⚪ P4 restant | #73 Sync (6-8j) | 6-8 jours | -| ⚪ P2 restant | #92 Assistant IA — écosystème d'outils phase 2 (web étendu, sources connectées, documents) | 3-5 jours | | ⚪ P0/P1 restant | #85-87 Refonte architecturale, performance, CI/CD (issues BUG-035 → BUG-040) | ~15-23 jours | -| **Total restant** | **10 items + finitions** | **~30-47 jours** | +| **Total restant** | **7 items + finitions** | **~27-42 jours** | --- diff --git a/docs/features/ai-tools-roadmap.md b/docs/features/ai-tools-roadmap.md index 33eb4a8..71d89c1 100644 --- a/docs/features/ai-tools-roadmap.md +++ b/docs/features/ai-tools-roadmap.md @@ -1,6 +1,6 @@ # #92 — Assistant IA — Écosystème d'outils : feuille de route technique -> **Statut :** ⚪ Backlog (phase 1 livrée dans #91) +> **Statut :** ✅ livré (phase 2, version 2.10.0) — phase 1 livrée dans #91 > **Effort estimé :** 3-5 jours pour la phase 2 | **Impact :** 🟠 > **Références :** [Roadmap](../ROADMAP.md) · [Outils & MCP #79](./ai-tools-mcp.md) · > [Fenêtre de discussion #91](./ai-assistant-conversation-ux.md) · [Changelog](../../CHANGELOG.md) @@ -36,6 +36,17 @@ réécriture de la boucle n'est nécessaire. ## 3. Phase 2 — catégories à implémenter +> **Livré (2.10.0, #92).** Récapitulatif des décisions finales : + +| Catégorie | Décision livrée | +|---|---| +| Recherche web étendue | Tavily, Brave, SerpAPI, Exa à clé (`OBSIGATE_*_API_KEY`), essayés avant SearXNG ; ordre via `OBSIGATE_WEB_PROVIDERS` | +| Lecture de pages | `fetch_url(render=True)` → worker Playwright isolé (`backend/tools/webrender.py`), dépendance optionnelle + erreur explicite | +| Crawl multi-pages | `crawl_site` (WRITE + confirmation) : BFS httpx borné (≤ 20 pages, même hôte, SSRF sur chaque URL) → condensé Markdown dans le vault. Scrapy écarté (dépendance lourde inutile à cette échelle) | +| Sources connectées | Gitea + GitHub (`git_list_repos`, `git_search_issues`, `git_get_file`) via env/Infisical ; drives cloud (Drive/OneDrive) orientés serveur MCP externe (#79) | +| Production de documents | `create_xlsx`, `create_docx`, `create_csv`, `create_pdf` — WRITE + confirmation, écrit via `save_raw_file(allow_docs=True)` (path safety + backup) | +| Transverse | Cache SQLite (`webcache.py`, TTL `OBSIGATE_WEB_CACHE_TTL`), retry backoff maison (`OBSIGATE_WEB_RETRY`), secrets par env (Infisical-compatible) | + ### 3.1 Recherche web étendue (`web_search`) - **Fallback sans clé — ✅ livré (BUG-051)** : chaîne de fournisseurs dans `backend/tools/web.py` — SearXNG auto-hébergé (`OBSIGATE_SEARXNG_URL`) puis, si diff --git a/frontend/locales/en.json b/frontend/locales/en.json index c6aeac0..b10b539 100644 --- a/frontend/locales/en.json +++ b/frontend/locales/en.json @@ -1822,6 +1822,14 @@ "ai.step.vaults": "Listed the vaults", "ai.step.fetch_url": "Opened a web page: {value}", "ai.step.web_search": "Searched the web: {value}", + "ai.step.crawl": "Crawled a site: {value}", + "ai.step.git_repos": "Listed repositories ({value})", + "ai.step.git_issues": "Searched issues: {value}", + "ai.step.git_file": "Read a repo file: {value}", + "ai.step.xlsx_create": "Spreadsheet proposed: {value}", + "ai.step.docx_create": "Word document proposed: {value}", + "ai.step.csv_create": "CSV file proposed: {value}", + "ai.step.pdf_create": "PDF document proposed: {value}", "bookslm.copied": "Copied to clipboard", "bookslm.error": "AI service error", "bookslm.regenerate": "Regenerate", diff --git a/frontend/locales/fr.json b/frontend/locales/fr.json index edbe6c1..8276761 100644 --- a/frontend/locales/fr.json +++ b/frontend/locales/fr.json @@ -1822,6 +1822,14 @@ "ai.step.vaults": "Liste des vaults consultée", "ai.step.fetch_url": "Page web consultée : {value}", "ai.step.web_search": "Recherche sur le web : {value}", + "ai.step.crawl": "Site exploré : {value}", + "ai.step.git_repos": "Dépôts listés ({value})", + "ai.step.git_issues": "Issues recherchées : {value}", + "ai.step.git_file": "Fichier de dépôt lu : {value}", + "ai.step.xlsx_create": "Tableur proposé : {value}", + "ai.step.docx_create": "Document Word proposé : {value}", + "ai.step.csv_create": "Fichier CSV proposé : {value}", + "ai.step.pdf_create": "Document PDF proposé : {value}", "bookslm.copied": "Réponse copiée dans le presse-papiers", "bookslm.error": "Erreur du service AI", "bookslm.regenerate": "Régénérer", diff --git a/package.json b/package.json index 181176c..1487075 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "obsigate", - "version": "2.9.1", + "version": "2.10.0", "description": "**Porte d'entrée web ultra-léger pour vos vaults Obsidian** — Accédez, naviguez et recherchez dans toutes vos notes Obsidian depuis n'importe quel appareil via une interface web moderne et responsive.", "main": "patch.js", "directories": { diff --git a/tests/conftest.py b/tests/conftest.py index 85b46f6..7a21a12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,23 @@ def _reset_tool_ratelimit(): ratelimit.reset() +@pytest.fixture(autouse=True) +def _disable_web_cache(): + """Web cache off by default: tests stay hermetic (no cross-test hits). + + tests/test_web_cache.py re-enables it explicitly with a tmp path. + """ + from backend.tools import webcache + + saved_path = os.environ.get("OBSIGATE_WEB_CACHE_PATH") + os.environ["OBSIGATE_WEB_CACHE_TTL"] = "0" + yield + if saved_path is None: + os.environ.pop("OBSIGATE_WEB_CACHE_PATH", None) + else: + os.environ["OBSIGATE_WEB_CACHE_PATH"] = saved_path + + @pytest.fixture(autouse=True) def _clean_env(): """Ensure no vault env vars leak between tests — but preserve test vault config.""" diff --git a/tests/test_connected_tools.py b/tests/test_connected_tools.py new file mode 100644 index 0000000..5aa81c1 --- /dev/null +++ b/tests/test_connected_tools.py @@ -0,0 +1,230 @@ +"""Unit tests for the connected sources (#92): Gitea & GitHub tools. + +All HTTP calls are mocked (httpx.request monkeypatched) — the CI never talks +to a real Gitea/GitHub instance. +""" + +import base64 +from typing import Any + +import pytest + +import backend.tools.connected as connected +from backend.tools.context import ToolContext, ToolError, ToolMode +from backend.tools.registry import get_tool + + +class FakeResponse: + def __init__(self, json_data: Any = None, status_code: int = 200): + self._json = json_data + self.status_code = status_code + + def json(self): + return self._json + + def raise_for_status(self): + if self.status_code >= 400: + import httpx + + raise httpx.HTTPStatusError("boom", request=None, response=self) # type: ignore[arg-type] + + +def _ctx() -> ToolContext: + return ToolContext(user={"username": "tester", "vaults": []}, mode=ToolMode.IN_APP) + + +@pytest.fixture +def gitea_env(monkeypatch): + monkeypatch.setenv("OBSIGATE_GITEA_URL", "https://git.example.net") + monkeypatch.setenv("OBSIGATE_GITEA_TOKEN", "tok-gitea") + + +@pytest.fixture +def github_env(monkeypatch): + monkeypatch.setenv("OBSIGATE_GITHUB_TOKEN", "tok-gh") + + +def _patch_request(monkeypatch, handler): + def fake_request(method, url, headers=None, timeout=None, follow_redirects=False, **kw): + captured = {"method": method, "url": str(url), "headers": headers or {}, + "params": kw.get("params")} + return handler(captured) + + monkeypatch.setattr(connected.httpx, "request", fake_request) + + +class TestRegistration: + @pytest.mark.parametrize("name", ["git_list_repos", "git_search_issues", "git_get_file"]) + def test_tools_registered_read(self, name): + from backend.tools.context import ToolRisk + + spec = get_tool(name) + assert spec is not None + assert spec.risk == ToolRisk.READ + assert "gitea" in spec.description or "github" in spec.description.lower() + + +class TestConfiguration: + def test_gitea_requires_base_url(self, monkeypatch): + monkeypatch.delenv("OBSIGATE_GITEA_URL", raising=False) + with pytest.raises(ToolError) as ei: + connected._provider_base("gitea") + assert ei.value.code == "provider_not_configured" + + def test_unknown_provider_rejected(self): + with pytest.raises(ToolError) as ei: + connected._provider_base("gitlab") + assert ei.value.code == "invalid_arguments" + + def test_gitea_token_sent_as_header(self, gitea_env, monkeypatch): + captured = {} + + def handler(captured_req): + captured.update(captured_req) + return FakeResponse(json_data={"data": []}) + + _patch_request(monkeypatch, handler) + connected.git_list_repos(_ctx(), connected.GitProviderInput(provider="gitea")) + assert captured["headers"]["Authorization"] == "token tok-gitea" + + +class TestListRepos: + def test_gitea_search_endpoint(self, gitea_env, monkeypatch): + captured = {} + + def handler(captured_req): + captured.update(captured_req) + return FakeResponse(json_data={"data": [ + {"name": "ObsiGate", "full_name": "bruno/ObsiGate", + "html_url": "https://git.example.net/bruno/ObsiGate", + "description": "vault gateway", "updated_at": "2026-09-01", + "private": False}, + ]}) + + _patch_request(monkeypatch, handler) + out = connected.git_list_repos(_ctx(), connected.GitProviderInput(provider="gitea")) + assert "/api/v1/repos/search" in captured["url"] + assert out["repos"][0]["name"] == "ObsiGate" + assert out["count"] == 1 + + def test_github_single_repo(self, github_env, monkeypatch): + captured = {} + + def handler(captured_req): + captured.update(captured_req) + return FakeResponse(json_data={ + "name": "ObsiGate", "full_name": "bruno/ObsiGate", + "html_url": "https://github.com/bruno/ObsiGate", + "description": "", "updated_at": "2026-09-02", "private": True, + }) + + _patch_request(monkeypatch, handler) + out = connected.git_list_repos(_ctx(), connected.GitProviderInput( + provider="github", repo="bruno/ObsiGate")) + assert captured["url"].endswith("/repos/bruno/ObsiGate") + assert out["repos"][0]["full_name"] == "bruno/ObsiGate" + assert out["repos"][0]["private"] is True + + +class TestSearchIssues: + def test_gitea_scoped_to_repo(self, gitea_env, monkeypatch): + captured = {} + + def handler(captured_req): + captured.update(captured_req) + return FakeResponse(json_data=[ + {"number": 12, "title": "Bug affichage", "html_url": "https://x/12", + "state": "open"}, + ]) + + _patch_request(monkeypatch, handler) + out = connected.git_search_issues(_ctx(), connected.GitSearchIssuesInput( + provider="gitea", query="affichage", repo="bruno/ObsiGate")) + assert "/repos/bruno/ObsiGate/issues" in captured["url"] + assert out["issues"][0]["id"] == 12 + assert out["issues"][0]["pull_request"] is False + + def test_github_search_syntax(self, github_env, monkeypatch): + captured = {} + + def handler(captured_req): + captured.update(captured_req) + return FakeResponse(json_data={"items": [ + {"number": 5, "title": "Crash on save", "html_url": "https://gh/5", + "state": "open", "pull_request": {"url": "x"}}, + ]}) + + _patch_request(monkeypatch, handler) + out = connected.git_search_issues(_ctx(), connected.GitSearchIssuesInput( + provider="github", query="crash", repo="bruno/ObsiGate", state="open")) + assert "/search/issues" in captured["url"] + assert "repo:bruno/ObsiGate" in captured["params"]["q"] + assert out["issues"][0]["pull_request"] is True + + +class TestGetFile: + def test_gitea_base64_content_decoded(self, gitea_env, monkeypatch): + payload = base64.b64encode("# Readme\n\nBonjour".encode()).decode() + + def handler(_captured): + return FakeResponse(json_data={ + "path": "README.md", "size": 15, "encoding": "base64", "content": payload, + }) + + _patch_request(monkeypatch, handler) + out = connected.git_get_file(_ctx(), connected.GitGetFileInput( + provider="gitea", repo="bruno/ObsiGate", path="README.md")) + assert "Bonjour" in out["content"] + assert out["truncated"] is False + + def test_github_ref_parameter(self, github_env, monkeypatch): + captured = {} + + def handler(captured_req): + captured.update(captured_req) + return FakeResponse(json_data={ + "path": "a.md", "size": 1, "encoding": "base64", + "content": base64.b64encode(b"x").decode(), + }) + + _patch_request(monkeypatch, handler) + connected.git_get_file(_ctx(), connected.GitGetFileInput( + provider="github", repo="o/r", path="a.md", ref="v2.9.0")) + assert captured["url"].endswith("?ref=v2.9.0") + + def test_missing_repo_or_path_rejected(self, gitea_env): + with pytest.raises(ToolError) as ei: + connected.git_get_file(_ctx(), connected.GitGetFileInput( + provider="gitea", repo="", path="a.md")) + assert ei.value.code == "invalid_arguments" + + +class TestErrors: + def test_404_maps_to_not_found(self, gitea_env, monkeypatch): + def handler(_captured): + return FakeResponse(json_data={}, status_code=404) + + _patch_request(monkeypatch, handler) + with pytest.raises(ToolError) as ei: + connected.git_list_repos(_ctx(), connected.GitProviderInput(provider="gitea")) + assert ei.value.code == "not_found" + + def test_401_maps_to_permission_denied(self, gitea_env, monkeypatch): + def handler(_captured): + return FakeResponse(json_data={}, status_code=401) + + _patch_request(monkeypatch, handler) + with pytest.raises(ToolError) as ei: + connected.git_list_repos(_ctx(), connected.GitProviderInput(provider="gitea")) + assert ei.value.code == "permission_denied" + + def test_network_error_maps_to_tool_error(self, gitea_env, monkeypatch): + import httpx + + def fake_request(*a, **kw): + raise httpx.ConnectError("down") + + monkeypatch.setattr(connected.httpx, "request", fake_request) + with pytest.raises(ToolError) as ei: + connected.git_list_repos(_ctx(), connected.GitProviderInput(provider="gitea")) + assert ei.value.code == "connected_source_unavailable" diff --git a/tests/test_crawler.py b/tests/test_crawler.py new file mode 100644 index 0000000..6050b47 --- /dev/null +++ b/tests/test_crawler.py @@ -0,0 +1,135 @@ +"""Unit tests for the bounded site crawler (#92): crawl_site (WRITE + confirmation).""" + +from typing import Any + +import pytest + +import backend.tools.crawler as crawler +from backend.tools.api import ToolConfirmationRequired, ToolContext, ToolError, call_tool + + +class FakeResponse: + def __init__(self, content: bytes = b"", status_code: int = 200, + headers: dict | None = None): + self.content = content + self.status_code = status_code + self.headers = headers or {"content-type": "text/html; charset=utf-8"} + self.encoding = "utf-8" + + def raise_for_status(self): + pass + + +def _ctx() -> ToolContext: + return ToolContext( + user={"username": "tester", "role": "admin", "vaults": ["*"]}, + audit_enabled=False, + ) + + +@pytest.fixture +def vault(tmp_path, monkeypatch): + vault_dir = tmp_path / "Vault" + vault_dir.mkdir() + monkeypatch.setitem(__import__("backend.indexer", fromlist=["index"]).index, + "Vault", {"name": "Vault", "path": str(vault_dir), "config": {}}) + return vault_dir + + +PAGE_A = ( + b"Docs" + b"

Bienvenue sur la documentation.

" + b'Suiteext' + b"" +) +PAGE_B = ( + b"Page B

Details ici.

" +) + + +@pytest.fixture +def local_urls(monkeypatch): + """Skip the DNS-based SSRF guard: test hosts are fake, HTTP is mocked.""" + monkeypatch.setattr(crawler, "_assert_public_http_url", lambda url: url) + + +@pytest.fixture +def two_pages(monkeypatch, local_urls): + def fake_get(url, **kw): + url = str(url) + if url.endswith("/page-b"): + return FakeResponse(content=PAGE_B) + return FakeResponse(content=PAGE_A) + + monkeypatch.setattr(crawler.httpx, "get", fake_get) + + +class TestConfirmation: + def test_requires_confirmation(self, vault, two_pages): + with pytest.raises(ToolConfirmationRequired): + call_tool("crawl_site", _ctx(), { + "url": "https://docs.example.dev/start", + "vault": "Vault", "path": "Crawls/docs.md", + }) + + +class TestCrawl: + def test_saves_same_host_pages(self, vault, two_pages): + out = call_tool("crawl_site", _ctx(), { + "url": "https://docs.example.dev/start", + "vault": "Vault", "path": "Crawls/docs.md", + }, confirm=True) + assert out.ok and out.data["pages"] == 2 + digest = (vault / "Crawls" / "docs.md").read_text(encoding="utf-8") + assert "# Crawl de docs.example.dev" in digest + assert "Bienvenue sur la documentation." in digest + assert "Details ici." in digest + assert "other.dev" not in digest + + def test_max_pages_bound(self, vault, monkeypatch, local_urls): + # A link farm: every page links to a new page — cap at max_pages. + def fake_get(url, **kw): + url = str(url) + n = int(url.rsplit("/", 1)[-1] or 0) + return FakeResponse( + content=f"P{n}" + f"

page {n}

next".encode()) + + monkeypatch.setattr(crawler.httpx, "get", fake_get) + out = call_tool("crawl_site", _ctx(), { + "url": "https://farm.example.dev/0", + "vault": "Vault", "path": "farm.md", "max_pages": 3, + }, confirm=True) + assert out.ok and out.data["pages"] == 3 + + def test_no_pages_recovered(self, vault, monkeypatch, local_urls): + def dead_get(*a, **kw): + raise crawler.httpx.ConnectError("down") + + monkeypatch.setattr(crawler.httpx, "get", dead_get) + with pytest.raises(ToolError) as ei: + call_tool("crawl_site", _ctx(), { + "url": "https://dead.example.dev/", "vault": "Vault", "path": "x.md", + }, confirm=True) + assert ei.value.code == "crawl_failed" + + def test_internal_url_rejected(self, vault): + with pytest.raises(ToolError) as ei: + call_tool("crawl_site", _ctx(), { + "url": "http://127.0.0.1:8080/", "vault": "Vault", "path": "x.md", + }, confirm=True) + assert ei.value.code in ("ssrf_blocked", "dns_error") + + def test_binary_content_skipped(self, vault, monkeypatch, local_urls): + def fake_get(url, **kw): + url = str(url) + if url.endswith("/x.pdf"): + return FakeResponse(content=b"%PDF-1.4", headers={"content-type": "application/pdf"}) + return FakeResponse(content=PAGE_A) + + monkeypatch.setattr(crawler.httpx, "get", fake_get) + out = call_tool("crawl_site", _ctx(), { + "url": "https://docs.example.dev/start", + "vault": "Vault", "path": "docs.md", "max_pages": 5, + }, confirm=True) + assert out.ok and out.data["pages"] >= 1 diff --git a/tests/test_document_tools.py b/tests/test_document_tools.py new file mode 100644 index 0000000..7ae92f6 --- /dev/null +++ b/tests/test_document_tools.py @@ -0,0 +1,160 @@ +"""Unit tests for the document-production tools (#92): create_xlsx, create_docx, +create_csv, create_pdf — WRITE risk, confirmation gating, vault persistence.""" + +import csv +import io +from pathlib import Path + +import pytest + +from backend.tools.api import ( + ToolConfirmationRequired, + ToolContext, + ToolError, + call_tool, + get_tool, +) +from backend.tools.context import ToolRisk + + +@pytest.fixture +def vault(tmp_path, monkeypatch): + """A minimal configured vault (index entry patched, no full build).""" + vault_dir = tmp_path / "Vault" + vault_dir.mkdir() + monkeypatch.setitem(__import__("backend.indexer", fromlist=["index"]).index, + "Vault", {"name": "Vault", "path": str(vault_dir), "config": {}}) + return vault_dir + + +def _ctx() -> ToolContext: + return ToolContext( + user={"username": "tester", "role": "admin", "vaults": ["*"]}, + audit_enabled=False, + ) + + +class TestRegistry: + @pytest.mark.parametrize("name", ["create_xlsx", "create_docx", "create_csv", "create_pdf"]) + def test_write_risk_and_confirmation(self, name): + spec = get_tool(name) + assert spec is not None + assert spec.risk == ToolRisk.WRITE + assert spec.requires_confirmation is True + + +class TestConfirmationGating: + def test_csv_requires_confirmation(self, vault): + with pytest.raises(ToolConfirmationRequired): + call_tool("create_csv", _ctx(), { + "vault": "Vault", "path": "data.csv", + "rows": [["a", "b"], [1, 2]], + }) + + def test_pdf_requires_confirmation(self, vault): + with pytest.raises(ToolConfirmationRequired): + call_tool("create_pdf", _ctx(), { + "vault": "Vault", "path": "doc.pdf", "title": "T", "content": "# H\npara", + }) + + +class TestCreateCsv: + def test_creates_file_in_vault(self, vault): + out = call_tool("create_csv", _ctx(), { + "vault": "Vault", "path": "Exports/data.csv", + "rows": [["nom", "score"], ["alice", 12], ["bob", 9.5]], + }, confirm=True) + assert out.ok and out.data["success"] is True + path = vault / "Exports" / "data.csv" + assert path.exists() + rows = list(csv.reader(io.StringIO(path.read_text(encoding="utf-8")))) + assert rows[0] == ["nom", "score"] + assert rows[2] == ["bob", "9.5"] + + def test_semicolon_delimiter(self, vault): + call_tool("create_csv", _ctx(), { + "vault": "Vault", "path": "d.csv", "delimiter": ";", + "rows": [["a", "b"], [1, 2]], + }, confirm=True) + content = (vault / "d.csv").read_text(encoding="utf-8") + assert "a;b" in content + + def test_wrong_extension_rejected(self, vault): + with pytest.raises(ToolError) as ei: + call_tool("create_csv", _ctx(), { + "vault": "Vault", "path": "d.txt", "rows": [["a"], [1]], + }, confirm=True) + assert ei.value.code == "invalid_arguments" + + def test_empty_rows_rejected(self, vault): + with pytest.raises(ToolError): + call_tool("create_csv", _ctx(), { + "vault": "Vault", "path": "d.csv", "rows": [], + }, confirm=True) + + +class TestCreateXlsx: + def test_creates_readable_workbook(self, vault): + call_tool("create_xlsx", _ctx(), { + "vault": "Vault", "path": "Rapports/budget.xlsx", + "rows": [["item", "cout"], ["serveur", 1200], ["licence", 300]], + "sheet_name": "Budget", + }, confirm=True) + from openpyxl import load_workbook + + wb = load_workbook(vault / "Rapports" / "budget.xlsx") + ws = wb.active + assert ws.title == "Budget" + assert ws.cell(row=1, column=1).value == "item" + assert ws.cell(row=2, column=2).value == 1200 + + def test_wrong_extension_rejected(self, vault): + with pytest.raises(ToolError): + call_tool("create_xlsx", _ctx(), { + "vault": "Vault", "path": "b.docx", "rows": [["a"], [1]], + }, confirm=True) + + +class TestCreateDocx: + def test_creates_readable_document(self, vault): + call_tool("create_docx", _ctx(), { + "vault": "Vault", "path": "rapport.docx", + "title": "Rapport hebdo", "paragraphs": ["Premier point.", "Second point."], + }, confirm=True) + import docx as docx_lib + + doc = docx_lib.Document(str(vault / "rapport.docx")) + texts = [p.text for p in doc.paragraphs] + assert "Rapport hebdo" in texts + assert "Second point." in texts + + def test_no_paragraphs_rejected(self, vault): + with pytest.raises(ToolError): + call_tool("create_docx", _ctx(), { + "vault": "Vault", "path": "r.docx", "paragraphs": [], + }, confirm=True) + + +class TestCreatePdf: + def test_creates_valid_pdf(self, vault): + call_tool("create_pdf", _ctx(), { + "vault": "Vault", "path": "docs/archi.pdf", + "title": "Architecture", "content": "# Titre\n\nUn paragraphe.\n## Sous-titre\nAutre texte.", + }, confirm=True) + raw = (vault / "docs" / "archi.pdf").read_bytes() + assert raw.startswith(b"%PDF") + + def test_long_content_truncated(self, vault): + call_tool("create_pdf", _ctx(), { + "vault": "Vault", "path": "big.pdf", "title": "T", "content": "x" * 500_000, + }, confirm=True) + assert (vault / "big.pdf").exists() + + +class TestVaultSafety: + def test_path_outside_vault_rejected(self, vault): + with pytest.raises(ToolError) as ei: + call_tool("create_csv", _ctx(), { + "vault": "Vault", "path": "../outside.csv", "rows": [["a"], [1]], + }, confirm=True) + assert ei.value.code in ("path_outside_vault", "invalid_arguments", "tool_execution_error") diff --git a/tests/test_web_cache.py b/tests/test_web_cache.py new file mode 100644 index 0000000..96db35f --- /dev/null +++ b/tests/test_web_cache.py @@ -0,0 +1,82 @@ +"""Unit tests for the SQLite web cache (backend.tools.webcache, #92).""" + +import pytest + +from backend.tools import webcache + + +@pytest.fixture +def cache_enabled(tmp_path, monkeypatch): + """Enable the cache against an isolated file with a short TTL.""" + monkeypatch.setenv("OBSIGATE_WEB_CACHE_PATH", str(tmp_path / "cache.sqlite3")) + monkeypatch.setenv("OBSIGATE_WEB_CACHE_TTL", "60") + webcache._schema_ready = False + yield + webcache._schema_ready = False + + +class TestCacheKey: + def test_deterministic_and_payload_sensitive(self): + a = webcache.cache_key("search", {"q": "pizza", "page": 1}) + b = webcache.cache_key("search", {"page": 1, "q": "pizza"}) + c = webcache.cache_key("search", {"q": "pasta", "page": 1}) + d = webcache.cache_key("fetch", {"q": "pizza", "page": 1}) + assert a == b + assert a != c + assert a != d + + +class TestCacheRoundTrip: + def test_set_get_roundtrip(self, cache_enabled): + key = webcache.cache_key("search", {"q": "x"}) + webcache.cache_set(key, {"results": [1, 2, 3], "provider": "tavily"}) + assert webcache.cache_get(key) == {"results": [1, 2, 3], "provider": "tavily"} + + def test_miss_returns_none(self, cache_enabled): + assert webcache.cache_get("search:unknown") is None + + def test_overwrite_updates_value(self, cache_enabled): + key = webcache.cache_key("search", {"q": "x"}) + webcache.cache_set(key, {"v": 1}) + webcache.cache_set(key, {"v": 2}) + assert webcache.cache_get(key) == {"v": 2} + + def test_ttl_expiry(self, tmp_path, monkeypatch): + monkeypatch.setenv("OBSIGATE_WEB_CACHE_PATH", str(tmp_path / "cache.sqlite3")) + monkeypatch.setenv("OBSIGATE_WEB_CACHE_TTL", "0.05") + webcache._schema_ready = False + key = webcache.cache_key("search", {"q": "x"}) + webcache.cache_set(key, {"v": 1}) + assert webcache.cache_get(key) == {"v": 1} + import time + + time.sleep(0.15) + assert webcache.cache_get(key) is None + + def test_disabled_when_ttl_zero(self, cache_enabled, monkeypatch): + monkeypatch.setenv("OBSIGATE_WEB_CACHE_TTL", "0") + key = webcache.cache_key("search", {"q": "x"}) + webcache.cache_set(key, {"v": 1}) + assert webcache.cache_get(key) is None + + def test_purge_expired(self, cache_enabled, monkeypatch): + key = webcache.cache_key("search", {"q": "x"}) + webcache.cache_set(key, {"v": 1}) + monkeypatch.setenv("OBSIGATE_WEB_CACHE_TTL", "0.05") + import time + + time.sleep(0.15) + assert webcache.purge_expired() >= 1 + assert webcache.cache_get(key) is None + + def test_corrupt_db_degrades_silently(self, tmp_path, monkeypatch): + # A directory as cache file breaks sqlite3.connect: the cache must + # disable itself instead of breaking the tools. + monkeypatch.setenv("OBSIGATE_WEB_CACHE_PATH", str(tmp_path)) + monkeypatch.setenv("OBSIGATE_WEB_CACHE_TTL", "60") + webcache._schema_ready = False + try: + assert webcache.cache_get("search:x") is None + webcache.cache_set("search:x", {"v": 1}) + finally: + webcache._schema_ready = False diff --git a/tests/test_web_search_providers.py b/tests/test_web_search_providers.py new file mode 100644 index 0000000..0b4c0cc --- /dev/null +++ b/tests/test_web_search_providers.py @@ -0,0 +1,155 @@ +"""Unit tests for the keyed web-search providers (#92): Tavily, Brave, +SerpAPI, Exa — plus provider ordering and the transient retry.""" + +import httpx +import pytest + +import backend.tools.web as web +from backend.tools.context import ToolContext, ToolError, ToolMode + + +class FakeResponse: + def __init__(self, json_data=None, status_code=200, content=b""): + self._json = json_data + self.status_code = status_code + self.content = content + self.encoding = "utf-8" + self.headers = {} + + def json(self): + return self._json + + def raise_for_status(self): + if self.status_code >= 400: + raise web.httpx.HTTPStatusError("boom", request=None, response=self) # type: ignore[arg-type] + + +def _ctx() -> ToolContext: + return ToolContext(user={"username": "tester", "vaults": []}, mode=ToolMode.IN_APP) + + +def _no_fallback(monkeypatch): + """Limit the chain to the provider under test (no searxng/ddg/bing noise).""" + monkeypatch.setattr(web, "WEB_FALLBACK_ENABLED", False) + monkeypatch.setattr(web, "SEARXNG_URL", "http://searxng.invalid") + monkeypatch.setattr(web.httpx, "get", lambda *a, **kw: (_ for _ in ()).throw( + httpx.ConnectError("offline"))) + monkeypatch.setattr(web.httpx, "post", lambda *a, **kw: (_ for _ in ()).throw( + httpx.ConnectError("offline"))) + + +class TestKeyedProviderParsers: + def test_tavily_maps_results(self, monkeypatch): + captured = {} + + def fake_post(url, json=None, **kw): + captured["url"] = url + captured["payload"] = json + return FakeResponse(json_data={"results": [ + {"title": "T", "url": "https://a.dev", "content": "c" * 800}, + ]}) + + monkeypatch.setattr(web.httpx, "post", fake_post) + results, engines = web._search_tavily("q", web.WebSearchInput(query="q", max_results=3)) + assert results[0]["title"] == "T" + assert len(results[0]["snippet"]) <= 600 + assert engines == [] + assert captured["payload"]["api_key"] == "" + assert captured["payload"]["max_results"] == 3 + + def test_brave_maps_results(self, monkeypatch): + captured = {} + + def fake_get(url, params=None, headers=None, **kw): + captured["url"] = url + captured["headers"] = headers + return FakeResponse(json_data={"web": {"results": [ + {"title": "B", "url": "https://b.dev", "description": "d"}, + ]}}) + + monkeypatch.setattr(web.httpx, "get", fake_get) + results, _ = web._search_brave("q", web.WebSearchInput(query="q")) + assert results[0]["title"] == "B" + assert "api.search.brave.com" in str(captured["url"]) + + def test_serpapi_maps_results(self, monkeypatch): + monkeypatch.setattr(web.httpx, "get", lambda *a, **kw: FakeResponse(json_data={ + "organic_results": [{"title": "S", "link": "https://s.dev", "snippet": "sn"}], + })) + results, _ = web._search_serpapi("q", web.WebSearchInput(query="q")) + assert results[0]["url"] == "https://s.dev" + + def test_exa_maps_results(self, monkeypatch): + monkeypatch.setattr(web.httpx, "post", lambda *a, **kw: FakeResponse(json_data={ + "results": [{"title": "E", "url": "https://e.dev", "text": "t" * 900}], + })) + results, _ = web._search_exa("q", web.WebSearchInput(query="q")) + assert results[0]["title"] == "E" + assert len(results[0]["snippet"]) <= 600 + + +class TestProviderChain: + def test_no_key_falls_back_to_searxng(self, monkeypatch): + for var in ("OBSIGATE_TAVILY_API_KEY", "OBSIGATE_BRAVE_API_KEY", + "OBSIGATE_SERPAPI_API_KEY", "OBSIGATE_EXA_API_KEY"): + monkeypatch.delenv(var, raising=False) + chain = [name for name, _ in web._provider_chain()] + assert chain[0] == "searxng" + assert "tavily" not in chain + + def test_keyed_provider_used_first_when_key_set(self, monkeypatch): + monkeypatch.setenv("OBSIGATE_TAVILY_API_KEY", "k") + chain = [name for name, _ in web._provider_chain()] + assert chain[0] == "tavily" + assert "brave" not in chain # no key → skipped + + def test_explicit_order_env(self, monkeypatch): + monkeypatch.setenv("OBSIGATE_TAVILY_API_KEY", "k") + monkeypatch.setenv("OBSIGATE_EXA_API_KEY", "k") + monkeypatch.setenv("OBSIGATE_WEB_PROVIDERS", "exa,unknown,tavily") + chain = [name for name, _ in web._provider_chain()] + assert chain[:2] == ["exa", "tavily"] + + def test_search_uses_keyed_provider_first(self, monkeypatch): + _no_fallback(monkeypatch) + monkeypatch.setenv("OBSIGATE_BRAVE_API_KEY", "k") + monkeypatch.setattr(web.httpx, "get", lambda *a, **kw: FakeResponse(json_data={ + "web": {"results": [{"title": "B", "url": "https://b.dev", "description": "d"}]}, + })) + out = web.web_search(_ctx(), web.WebSearchInput(query="q")) + assert out["provider"] == "brave" + assert out["count"] == 1 + + +class TestRetry: + def test_transient_error_retried_then_succeeds(self, monkeypatch): + monkeypatch.setattr(web, "WEB_RETRY_ATTEMPTS", 1) + monkeypatch.setattr(web, "_provider_chain", lambda: [("searxng", web._search_searxng)]) + calls = {"n": 0} + + def flaky_get(*a, **kw): + calls["n"] += 1 + if calls["n"] == 1: + raise httpx.ConnectError("blip") + return FakeResponse(json_data={"results": [ + {"title": "A", "url": "https://a.dev", "content": "x"}]}) + + monkeypatch.setattr(web.httpx, "get", flaky_get) + out = web.web_search(_ctx(), web.WebSearchInput(query="q")) + assert out["provider"] == "searxng" + assert calls["n"] == 2 + + def test_persistent_error_not_retried_forever(self, monkeypatch): + monkeypatch.setattr(web, "WEB_RETRY_ATTEMPTS", 1) + monkeypatch.setattr(web, "_provider_chain", lambda: [("searxng", web._search_searxng)]) + calls = {"n": 0} + + def dead_get(*a, **kw): + calls["n"] += 1 + raise httpx.ConnectError("down") + + monkeypatch.setattr(web.httpx, "get", dead_get) + with pytest.raises(ToolError) as ei: + web.web_search(_ctx(), web.WebSearchInput(query="q")) + assert ei.value.code == "web_search_unavailable" + assert calls["n"] == 2 # initial + 1 retry, per provider diff --git a/tests/test_webrender.py b/tests/test_webrender.py new file mode 100644 index 0000000..c0df20d --- /dev/null +++ b/tests/test_webrender.py @@ -0,0 +1,73 @@ +"""Unit tests for the dynamic rendering path (#92): fetch_url(render=True).""" + +import pytest + +import backend.tools.web as web +from backend.tools import webrender +from backend.tools.context import ToolContext, ToolError, ToolMode +from backend.tools.registry import get_tool + + +def _ctx() -> ToolContext: + return ToolContext(user={"username": "tester", "vaults": []}, mode=ToolMode.IN_APP) + + +class TestRegistration: + def test_render_param_exposed_in_schema(self): + spec = get_tool("fetch_url") + assert spec is not None + assert "render" in spec.input_model.model_fields + + +class TestRenderUnavailable: + def test_missing_playwright_clear_error(self, monkeypatch): + monkeypatch.setattr(webrender, "_playwright_available", lambda: False) + with pytest.raises(ToolError) as ei: + web.fetch_url(_ctx(), web.FetchUrlInput( + url="https://example.com/spa", render=True)) + assert ei.value.code == "playwright_unavailable" + + def test_ssrf_guard_applied_before_render(self, monkeypatch): + monkeypatch.setattr(webrender, "_playwright_available", lambda: True) + with pytest.raises(ToolError) as ei: + web.fetch_url(_ctx(), web.FetchUrlInput( + url="http://127.0.0.1:9222/devtools", render=True)) + assert ei.value.code in ("ssrf_blocked", "dns_error") + + +class TestRenderSuccess: + def test_fetch_url_delegates_to_worker(self, monkeypatch): + captured = {} + + def fake_render(url): + captured["url"] = url + return {"url": url, "status": 200, "title": "SPA", + "text": "dynamic content", "rendered": True, "truncated": False} + + monkeypatch.setattr(webrender, "render_page", fake_render) + out = web.fetch_url(_ctx(), web.FetchUrlInput( + url="https://example.com/spa", render=True)) + assert captured["url"] == "https://example.com/spa" + assert out["rendered"] is True + assert "dynamic content" in out["text"] + + def test_worker_failure_maps_to_tool_error(self, monkeypatch): + monkeypatch.setattr(webrender, "_playwright_available", lambda: True) + + def boom(url): + raise RuntimeError("chromium crashed") + + # The executor re-raises the worker exception on .result(); render_page + # must wrap it into a ToolError instead of leaking a bare exception. + monkeypatch.setattr(webrender, "_render_in_worker", boom) + with pytest.raises(ToolError) as ei: + web.fetch_url(_ctx(), web.FetchUrlInput( + url="https://example.com/spa", render=True)) + assert ei.value.code == "render_unavailable" + + +class TestMarkdownExtraction: + def test_html_to_text_reused(self): + text = webrender._html_to_text("

hello

") + assert "hello" in text + assert "x()" not in text