diff --git a/CHANGELOG.md b/CHANGELOG.md index 354ac9f..5e3d2d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,17 @@ et [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Ajouté +- **#79 Phase C — Catalogue d'outils lecture & recherche** — 10 nouveaux outils IA exposés par la + couche partagée (`backend/tools/service.py`) : `list_all_files`, `read_file_raw`, `get_backlinks`, + `list_backups`, `diff_backup`, `get_graph`, `search_advanced`, `search_paths`, `suggest_tags`, + `list_recent`. Nouveaux services réutilisables `backend/services/backups.py`, `graph.py`, + `recent.py` et extensions de `vaults.py` (`list_all_files`) / `search.py` + (`advanced_search_vaults`, `search_paths`). Les routes `/api/recent`, `/api/vault/{vault}/files`, + `/api/file/{vault}/backups`, `/api/file/{vault}/diff`, `/api/graph/{vault}`, `/api/tree-search` + et `/api/search/advanced` délèguent désormais à ces services (source unique de vérité). + Filtrage systématique par permissions vault ; secrets redactés sur `read_file`/`read_file_raw`. + Tests : `tests/test_tools.py` (+17 tests). + - **#79 Assistant IA — Outils (function calling) & serveur MCP — Phase 0 (couche d'outils partagée)** — nouveau paquet `backend/tools/` : `context.py` (`ToolContext`), `registry.py` (décorateur `@tool` + schéma JSON), `schemas.py`, `service.py`, `audit.py` (journalisation JSONL des appels), façade diff --git a/backend/main.py b/backend/main.py index 3ec2864..4ea0b40 100644 --- a/backend/main.py +++ b/backend/main.py @@ -28,8 +28,6 @@ from starlette.middleware.base import BaseHTTPMiddleware from backend.attachment_indexer import get_attachment_stats, rescan_vault_attachments from backend.history import ( get_bookmarks, - get_recent_opened, - is_bookmarked, record_open, remove_recent, toggle_bookmark, @@ -38,7 +36,6 @@ from backend.history import ( ) from backend.image_processor import preprocess_images from backend.indexer import ( - IGNORED_DIRS, SUPPORTED_EXTENSIONS, _extract_tags, add_vault_to_index, @@ -50,7 +47,6 @@ from backend.indexer import ( handle_file_move, index, parse_markdown_file, - path_index, reload_index, remove_single_file, remove_vault_from_index, @@ -103,11 +99,20 @@ from backend.search import ( suggest_tags, suggest_titles, ) +from backend.services.backups import diff_backup as service_diff_backup +from backend.services.backups import get_backup_dir as service_get_backup_dir +from backend.services.backups import list_backup_files as service_list_backup_files from backend.services.errors import ServiceError from backend.services.files import read_raw_file +from backend.services.graph import get_graph as service_get_graph +from backend.services.recent import humanize_mtime, list_recent +from backend.services.search import advanced_search_vaults, search_paths, search_vaults from backend.services.search import list_tags as service_list_tags -from backend.services.search import search_vaults -from backend.services.vaults import browse_directory, list_accessible_vaults +from backend.services.vaults import ( + browse_directory, + list_accessible_vaults, + list_all_files, +) from backend.vault_settings import ( get_vault_setting, update_vault_setting, @@ -1272,111 +1277,21 @@ async def api_vaults(current_user=Depends(require_auth)): return list_accessible_vaults(current_user) -def humanize_mtime(mtime: float) -> str: - delta = time.time() - mtime - if delta < 60: - return "à l'instant" - if delta < 3600: - return f"il y a {int(delta/60)} min" - if delta < 86400: - return f"il y a {int(delta/3600)} h" - if delta < 604800: - return f"il y a {int(delta/86400)} j" - return datetime.fromtimestamp(mtime).strftime("%d %b %Y") - - @app.get("/api/recent", response_model=RecentResponse) async def api_recent(limit: int | None = Query(None), vault: str | None = Query(None), mode: str | None = Query("opened"), current_user=Depends(require_auth)): config = _load_config() actual_limit = limit if limit is not None else config.get("recent_files_limit", 20) - + username = current_user.get("username") user_vaults = current_user.get("_token_vaults") or current_user.get("vaults", []) - - if mode == "opened" and username: - # Use history file for "last opened" - history = get_recent_opened(username, vault_filter=vault, limit=actual_limit) - files_resp = [] - for item in history: - v_name = item["vault"] - if "*" not in user_vaults and v_name not in user_vaults: - continue - - # Find in index to get metadata/preview - f_idx = find_file_in_index(item["path"], v_name) - if f_idx: - files_resp.append({ - "path": f_idx["path"], - "title": f_idx.get("title") or item["path"].split("/")[-1], - "vault": v_name, - "mtime": item["opened_at"], - "mtime_human": humanize_mtime(item["opened_at"]), - "size_bytes": f_idx.get("size", 0), - "tags": [f"#{t}" for t in f_idx.get("tags", [])][:5], - "preview": f_idx.get("content_preview", "")[:120], - "bookmarked": is_bookmarked(username, v_name, f_idx["path"]) - }) - else: - # File might have been renamed/deleted since last open - files_resp.append({ - "path": item["path"], - "title": item.get("title") or item["path"].split("/")[-1], - "vault": v_name, - "mtime": item["opened_at"], - "mtime_human": humanize_mtime(item["opened_at"]), - "tags": [], - "preview": "", - "bookmarked": is_bookmarked(username, v_name, item["path"]) - }) - return { - "files": files_resp, - "total": len(files_resp), - "limit": actual_limit, - "mode": "opened" - } - # Fallback to "last modified" (original logic) - all_files = [] - for v_name, v_data in index.items(): - if vault and v_name != vault: - continue - if "*" not in user_vaults and v_name not in user_vaults: - continue - for f in v_data.get("files", []): - all_files.append((v_name, f)) - - # Sort descending by ISO string "modified" - all_files.sort(key=lambda x: x[1].get("modified", ""), reverse=True) - recent = all_files[:actual_limit] - - files_resp = [] - for v_name, f in recent: - iso_modified = f.get("modified", "") - try: - mtime_dt = datetime.fromisoformat(iso_modified.replace("Z", "+00:00")) - mtime_val = mtime_dt.timestamp() - except Exception: - mtime_val = time.time() - - files_resp.append({ - "path": f["path"], - "title": f["title"], - "vault": v_name, - "mtime": mtime_val, - "mtime_human": humanize_mtime(mtime_val), - "mtime_iso": iso_modified, - "size_bytes": f.get("size", 0), - "tags": [f"#{t}" for t in f.get("tags", [])][:5], - "preview": f.get("content_preview", "")[:120], - "bookmarked": is_bookmarked(username, v_name, f["path"]) - }) - - return { - "files": files_resp, - "total": len(all_files), - "limit": actual_limit, - "mode": "modified" - } + return list_recent( + username, + user_vaults, + vault=vault, + limit=actual_limit, + mode=mode or "opened", + ) @app.get("/api/bookmarks", response_model=BookmarksResponse) @@ -2315,73 +2230,20 @@ async def api_file_move( def _get_backup_dir(vault_name: str, relative_path: str) -> Path: """Return the directory where backups for a specific file are stored. - - Resolves relative backup paths against the SPECIFIC vault's directory, - matching _backup_file's resolution logic exactly. + + Thin wrapper around :func:`backend.services.backups.get_backup_dir` + (single implementation used by both routes and tools). """ - backup_root = Path(os.environ.get("OBSIGATE_BACKUP_DIR", ".obsigate-backup")) - if not backup_root.is_absolute(): - vault_data = get_vault_data(vault_name) - if vault_data: - backup_root = Path(vault_data["path"]) / backup_root - return backup_root / vault_name / Path(relative_path).parent + return service_get_backup_dir(vault_name, relative_path) def _list_backup_files(vault_name: str, relative_path: str) -> list[dict]: """List all backup files for a given vault file, sorted newest first. - Backup filename format: {original_filename}.{timestamp}.bak - - Returns a list of dicts with timestamp, datetime, size, filename. + Thin wrapper around :func:`backend.services.backups.list_backup_files` + (single implementation used by both routes and tools). """ - backup_dir = _get_backup_dir(vault_name, relative_path) - if not backup_dir.exists(): - return [] - - original_name = Path(relative_path).name - prefix = original_name + "." - backups: list[dict[str, Any]] = [] - - try: - dir_entries = list(backup_dir.iterdir()) - except PermissionError: - logger.warning(f"Permission denied reading backup dir: {backup_dir}") - return [] - except OSError as e: - logger.error(f"Error reading backup dir {backup_dir}: {e}") - return [] - - for f in dir_entries: - try: - if not f.is_file(): - continue - name = f.name - if not name.startswith(prefix) or not name.endswith(".bak"): - continue - # Extract timestamp: strip prefix and .bak suffix - ts_part = name[len(prefix):-len(".bak")] - try: - ts = int(ts_part) - except (ValueError, TypeError): - continue - try: - dt = datetime.fromtimestamp(ts, tz=timezone.utc).isoformat() - except (OSError, OverflowError, ValueError) as ts_err: - logger.warning(f"Skipping backup with invalid timestamp {ts}: {ts_err}") - continue - st = f.stat() - backups.append({ - "timestamp": ts, - "datetime": dt, - "size": st.st_size, - "filename": name, - }) - except Exception as entry_err: - logger.warning(f"Skipping unreadable backup entry {f}: {entry_err}") - continue - - backups.sort(key=lambda b: b["timestamp"], reverse=True) - return backups + return service_list_backup_files(vault_name, relative_path) @app.get("/api/file/{vault_name}/backups", response_model=BackupsResponse) @@ -2444,69 +2306,7 @@ async def api_file_diff( if not check_vault_access(vault_name, current_user): raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'") - vault_data = get_vault_data(vault_name) - if not vault_data: - raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found") - - vault_root = Path(vault_data["path"]) - file_path = _resolve_safe_path(vault_root, path) - - if not file_path.exists() or not file_path.is_file(): - raise HTTPException(status_code=404, detail=f"File not found: {path}") - - original_name = Path(path).name - - def _read_backup(ts: int) -> tuple[str, str]: - """Read a backup file. Returns (content, label).""" - backup_dir = _get_backup_dir(vault_name, path) - backup_path = backup_dir / f"{original_name}.{ts}.bak" - if not backup_path.exists(): - raise HTTPException(status_code=404, detail=f"Backup version {ts} not found for {path}") - try: - content = backup_path.read_text(encoding="utf-8", errors="replace") - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to read backup {ts}: {e}") - dt = datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") - return content, f"{path}@{dt}" - - try: - # Read left side (backup version) - left_content, left_label = _read_backup(version) - - # Read right side - if compare_with is not None: - right_content, right_label = _read_backup(compare_with) - else: - try: - right_content = file_path.read_text(encoding="utf-8", errors="replace") - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to read current file: {e}") - right_label = f"{path} (current)" - - # Generate unified diff using difflib - import difflib - left_lines = left_content.splitlines(keepends=True) - right_lines = right_content.splitlines(keepends=True) - - diff_lines = list(difflib.unified_diff( - left_lines, right_lines, - fromfile=left_label, tofile=right_label, - )) - - return { - "vault": vault_name, - "path": path, - "version": version, - "compare_with": compare_with, - "diff": "".join(diff_lines), - "left_content": left_content, - "right_content": right_content, - } - except HTTPException: - raise - except Exception as e: - logger.error(f"Error generating diff for {vault_name}/{path}: {type(e).__name__}: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Erreur lors de la génération du diff: {e!s}") + return service_diff_backup(vault_name, path, version, compare_with) @app.post("/api/file/{vault_name}/restore", response_model=RestoreResponse) @@ -3069,31 +2869,7 @@ async def api_tree_search( Returns: ``TreeSearchResponse`` with matching paths. """ - if not q: - return {"query": q, "vault_filter": vault, "results": []} - - query_lower = q.lower() - results = [] - - vaults_to_search = [vault] if vault != "all" else list(path_index.keys()) - - for vault_name in vaults_to_search: - vault_paths = path_index.get(vault_name, []) - - for entry in vault_paths: - path_lower = entry["path"].lower() - name_lower = entry["name"].lower() - - if query_lower in name_lower or query_lower in path_lower: - results.append({ - "vault": vault_name, - "path": entry["path"], - "name": entry["name"], - "type": entry["type"], - "matched_path": entry["path"], - }) - - return {"query": q, "vault_filter": vault, "results": results} + return search_paths(q, vault) @app.get("/api/search/advanced", response_model=AdvancedSearchResponse) @@ -3134,8 +2910,8 @@ async def api_advanced_search( loop = asyncio.get_event_loop() return await loop.run_in_executor( _search_executor, - partial(advanced_search, q, vault_filter=vault, tag_filter=tag, - limit=limit, offset=offset, sort_by=sort, + partial(advanced_search_vaults, q, vault=vault, tag=tag, + limit=limit, offset=offset, sort=sort, case_sensitive=case_sensitive, whole_word=whole_word, regex=regex, include_paths=include_paths, exclude_paths=exclude_paths, created=created, modified=modified, size=size), @@ -3318,175 +3094,7 @@ async def api_graph( if not check_vault_access(vault_name, current_user): raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'") - vault_data = get_vault_data(vault_name) - if not vault_data: - raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found") - - vault_root = Path(vault_data["path"]) - target = _resolve_safe_path(vault_root, path) if path else vault_root.resolve() - - if not target.exists(): - raise HTTPException(status_code=404, detail=f"Path not found: {path}") - - nodes: list[dict] = [] - edges: list[dict] = [] - node_ids: set = set() - - def _add_node(name: str, ntype: str, npath: str, size: int = 0, - tags: list[str] | None = None, incoming: int = 0, outgoing: int = 0) -> str: - nid = f"{vault_name}:{npath}" - if nid not in node_ids: - node_ids.add(nid) - nodes.append({ - "id": nid, "name": name, "type": ntype, "path": npath, - "size": size, "tags": tags or [], - "incoming_count": incoming, "outgoing_count": outgoing, - }) - return nid - - def _add_edge(source: str, target: str, relation: str): - edges.append({"source": source, "target": target, "relation": relation}) - - # Get vault settings for hidden files - from backend.vault_settings import get_vault_setting - settings = get_vault_setting(vault_name) or {} - hide_hidden = settings.get("hideHiddenFiles", False) - - # Build tag index from the in-memory index for fast lookups - _tag_index: dict[str, list[str]] = {} - for doc_key, info in index.items(): - vn, fp = doc_key.split("::", 1) if "::" in doc_key else ("", "") - if vn == vault_name: - for t in info.get("tags", []): - _tag_index.setdefault(t.lower(), []).append(fp) - - # Determine scope - if scope == "full": - # Full vault — walk entire vault root, ignore path param - target = vault_root.resolve() - effective_depth = depth if depth > 0 else 2 # minimum depth 2 for full view - else: - target = _resolve_safe_path(vault_root, path) if path else vault_root.resolve() - effective_depth = depth - - if not target.exists(): - raise HTTPException(status_code=404, detail=f"Path not found: {path}") - - # Add the focus node - focus_name = path.split("/")[-1] if path else vault_name - focus_type = "directory" if path else "vault" - focus_id = _add_node(focus_name, focus_type, path) - - # Walk directory tree up to depth levels - def _walk_dir(dir_path: Path, parent_id: str, current_depth: int): - if current_depth > effective_depth: - return - try: - for entry in sorted(dir_path.iterdir(), key=lambda e: (not e.is_dir(), e.name.lower())): - if hide_hidden and entry.name.startswith("."): - continue - rel = str(entry.relative_to(vault_root)).replace("\\", "/") - - # Tag filter: skip files that don't have the requested tag - if tag and entry.is_file(): - file_tags = [t.lower() for t in _tag_index.get(rel, [])] - if tag.lower() not in file_tags: - continue - - if entry.is_dir(): - did = _add_node(entry.name, "directory", rel) - _add_edge(parent_id, did, "parent") - if current_depth < effective_depth: - _walk_dir(entry, did, current_depth + 1) - elif entry.suffix.lower() in SUPPORTED_EXTENSIONS or entry.name.lower() in ("dockerfile", "makefile"): - file_tags = _tag_index.get(rel, []) - fid = _add_node(entry.name, "file", rel, entry.stat().st_size, tags=file_tags) - _add_edge(parent_id, fid, "parent") - except PermissionError: - pass - - if target.is_dir(): - _walk_dir(target, focus_id, 0) - elif target.is_file(): - _walk_dir(target.parent, focus_id, 0) - - # Add wikilink edges between markdown files in the current scope - _add_wikilink_edges(nodes, edges, node_ids, vault_name) - - # Compute incoming/outgoing counts from edges - edge_counts: dict[str, dict[str, int]] = {} - for node in nodes: - edge_counts[node["id"]] = {"incoming": 0, "outgoing": 0} - for edge in edges: - if edge["relation"] in ("wikilink", "backlink"): - src = edge["source"] - tgt = edge["target"] - if src in edge_counts: - edge_counts[src]["outgoing"] += 1 - if tgt in edge_counts: - edge_counts[tgt]["incoming"] += 1 - for node in nodes: - counts = edge_counts.get(node["id"], {"incoming": 0, "outgoing": 0}) - node["incoming_count"] = counts["incoming"] - node["outgoing_count"] = counts["outgoing"] - - return {"vault": vault_name, "path": path, "scope": scope, - "nodes": nodes, "edges": edges} - - -def _add_wikilink_edges(nodes: list, edges: list, node_ids: set, vault_name: str): - """Add edges for wikilinks between markdown files in the current graph scope.""" - - # Only consider files nodes - file_nodes = [n for n in nodes if n["type"] == "file" and n["path"].endswith(".md")] - if len(file_nodes) < 2: - return - - # Build lookup: relative_path → node_id - path_to_id = {n["path"]: n["id"] for n in file_nodes} - - wikilink_pattern = re.compile(r"\[\[([^\]|#]+)(?:[|#][^\]]+)?\]\]") - - for node in file_nodes: - # Get the file content from index - vault_data = index.get(vault_name) - if not vault_data: - continue - file_entry = None - for f in vault_data.get("files", []): - if f["path"] == node["path"]: - file_entry = f - break - if not file_entry: - continue - - content = file_entry.get("content", "") - if not content: - continue - - # Find all wikilinks in content - for match in wikilink_pattern.finditer(content): - target = match.group(1).strip() - # Try to find the target in our graph scope first - target_lower = target.lower() - if not target_lower.endswith(".md"): - target_lower += ".md" - - for target_path, target_id in path_to_id.items(): - if target_id == node["id"]: - continue - target_name = target_path.rsplit("/", 1)[-1].lower() - if target_name == target_lower or target_path.lower() == target_lower: - # Avoid duplicate edges - edge_key = tuple(sorted([node["id"], target_id])) - if edge_key not in {(e["source"], e["target"]) for e in edges} and \ - edge_key not in {(e["target"], e["source"]) for e in edges}: - edges.append({ - "source": node["id"], - "target": target_id, - "relation": "wikilink" - }) - break + return service_get_graph(vault_name, path=path, depth=depth, scope=scope, tag=tag) @app.get("/api/index/reload/{vault_name}", response_model=VaultStatsResponse) @@ -3814,85 +3422,7 @@ async def api_vault_recent_files( if not check_vault_access(vault_name, current_user): raise HTTPException(status_code=403, detail=f"Accès refusé à la vault '{vault_name}'") - vault_data = get_vault_data(vault_name) - if not vault_data: - raise HTTPException(status_code=404, detail=f"Vault '{vault_name}' not found") - - vault_root = Path(vault_data["path"]) - dir_path = _resolve_safe_path(vault_root, dir) if dir else vault_root - - if not dir_path.exists() or not dir_path.is_dir(): - raise HTTPException(status_code=404, detail=f"Directory not found: {dir}") - - files: list[dict[str, Any]] = [] - dir_prefix = (dir or "").strip("/") - - try: - iterator = dir_path.rglob('*') if recursive else dir_path.iterdir() - for entry in iterator: - if not entry.is_file(): - continue - if entry.name.startswith('.'): - continue - if entry.name in IGNORED_DIRS: - continue - - # Skip entries whose parent dir chain contains an ignored dir (for rglob) - if recursive and dir_prefix: - skip = False - try: - for part in entry.relative_to(dir_path).parts[:-1]: - if part.startswith('.') or part in IGNORED_DIRS: - skip = True - break - except ValueError: - pass - if skip: - continue - - stat = entry.stat() - rel_path = str(entry.relative_to(vault_root)).replace("\\", "/") - - # Compute relative directory from the requested dir - if recursive and dir_prefix: - try: - rel_to_dir = str(entry.parent.relative_to(dir_path)).replace("\\", "/") - except ValueError: - rel_to_dir = "" - else: - rel_to_dir = "" - - ext = entry.suffix.lower() if entry.suffix else "" - - file_entry = { - "name": entry.name, - "path": rel_path, - "vault": vault_name, - "size": stat.st_size, - "modified": stat.st_mtime, - "modified_iso": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(), - "extension": ext.lstrip(".") if ext else "", - } - if rel_to_dir and rel_to_dir != ".": - file_entry["rel_dir"] = rel_to_dir - - files.append(file_entry) - except PermissionError: - raise HTTPException(status_code=403, detail="Permission denied reading directory") - - # Sort by modification time, newest first - files.sort(key=lambda f: f["modified"], reverse=True) - - # Apply limit - files = files[:limit] - - return { - "vault": vault_name, - "directory": dir, - "recursive": recursive, - "count": len(files), - "files": files, - } + return list_all_files(vault_name, dir=dir, limit=limit, recursive=recursive) @app.get("/api/vaults/settings/all", response_model=AllVaultSettingsResponse) diff --git a/backend/services/backups.py b/backend/services/backups.py new file mode 100644 index 0000000..01ee019 --- /dev/null +++ b/backend/services/backups.py @@ -0,0 +1,195 @@ +"""Backup inspection services shared by REST routes and the AI tool layer. + +Single source of truth for locating, listing and diffing the timestamped +backups created before each file mutation. The write-side (creating a backup) +stays in the route layer; only the read/inspection logic lives here so both +``/api/file/{vault}/backups``, ``/api/file/{vault}/diff`` and the tools +``list_backups`` / ``diff_backup`` behave identically. +""" + +from __future__ import annotations + +import difflib +import logging +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from backend.services.errors import ServiceError +from backend.services.paths import resolve_safe_path +from backend.services.vaults import get_vault_root + +logger = logging.getLogger("obsigate.services.backups") + + +def get_backup_dir(vault_name: str, relative_path: str) -> Path: + """Return the directory where backups for a specific file are stored. + + Resolves relative backup paths against the SPECIFIC vault's directory, + matching the backup-creation logic exactly. + """ + from backend.indexer import get_vault_data + + backup_root = Path(os.environ.get("OBSIGATE_BACKUP_DIR", ".obsigate-backup")) + if not backup_root.is_absolute(): + vault_data = get_vault_data(vault_name) + if vault_data: + backup_root = Path(vault_data["path"]) / backup_root + return backup_root / vault_name / Path(relative_path).parent + + +def list_backup_files(vault_name: str, relative_path: str) -> list[dict[str, Any]]: + """List all backup files for a vault file, sorted newest first. + + Backup filename format: ``{original_filename}.{timestamp}.bak``. + + Returns a list of dicts with ``timestamp``, ``datetime``, ``size`` and + ``filename``. Missing or unreadable backup directories yield an empty list. + """ + backup_dir = get_backup_dir(vault_name, relative_path) + if not backup_dir.exists(): + return [] + + original_name = Path(relative_path).name + prefix = original_name + "." + backups: list[dict[str, Any]] = [] + + try: + dir_entries = list(backup_dir.iterdir()) + except PermissionError: + logger.warning(f"Permission denied reading backup dir: {backup_dir}") + return [] + except OSError as e: + logger.error(f"Error reading backup dir {backup_dir}: {e}") + return [] + + for f in dir_entries: + try: + if not f.is_file(): + continue + name = f.name + if not name.startswith(prefix) or not name.endswith(".bak"): + continue + ts_part = name[len(prefix):-len(".bak")] + try: + ts = int(ts_part) + except (ValueError, TypeError): + continue + try: + dt = datetime.fromtimestamp(ts, tz=timezone.utc).isoformat() + except (OSError, OverflowError, ValueError) as ts_err: + logger.warning(f"Skipping backup with invalid timestamp {ts}: {ts_err}") + continue + st = f.stat() + backups.append({ + "timestamp": ts, + "datetime": dt, + "size": st.st_size, + "filename": name, + }) + except Exception as entry_err: + logger.warning(f"Skipping unreadable backup entry {f}: {entry_err}") + continue + + backups.sort(key=lambda b: b["timestamp"], reverse=True) + return backups + + +def diff_backup( + vault_name: str, + path: str, + version: int, + compare_with: int | None = None, +) -> dict[str, Any]: + """Generate a unified diff between a backup version and another version or the current file. + + Args: + vault_name: Name of the vault. + path: Relative path of the file within the vault. + version: Timestamp of the backup used as the old/left side. + compare_with: Optional timestamp of another backup as the new/right + side. If omitted, the current file on disk is used. + + Returns: + Dict with ``vault``, ``path``, ``version``, ``compare_with``, ``diff``, + ``left_content`` and ``right_content``. + + Raises: + ServiceError: ``not_found`` (404) when the file or a backup is missing, + ``read_error`` (500) when a file cannot be read. + """ + root = get_vault_root(vault_name) + file_path = resolve_safe_path(root, path) + + if not file_path.exists() or not file_path.is_file(): + raise ServiceError( + f"File not found: {path}", + code="not_found", + status=404, + details={"vault": vault_name, "path": path}, + ) + + original_name = Path(path).name + + def _read_backup(ts: int) -> tuple[str, str]: + backup_dir = get_backup_dir(vault_name, path) + backup_path = backup_dir / f"{original_name}.{ts}.bak" + if not backup_path.exists(): + raise ServiceError( + f"Backup version {ts} not found for {path}", + code="not_found", + status=404, + details={"vault": vault_name, "path": path, "version": ts}, + ) + try: + content = backup_path.read_text(encoding="utf-8", errors="replace") + except Exception as e: + raise ServiceError( + f"Failed to read backup {ts}: {e}", + code="read_error", + status=500, + details={"vault": vault_name, "path": path, "version": ts}, + ) from e + dt = datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + return content, f"{path}@{dt}" + + try: + left_content, left_label = _read_backup(version) + + if compare_with is not None: + right_content, right_label = _read_backup(compare_with) + else: + try: + right_content = file_path.read_text(encoding="utf-8", errors="replace") + except Exception as e: + raise ServiceError( + f"Failed to read current file: {e}", + code="read_error", + status=500, + details={"vault": vault_name, "path": path}, + ) from e + right_label = f"{path} (current)" + + left_lines = left_content.splitlines(keepends=True) + right_lines = right_content.splitlines(keepends=True) + + diff_lines = list(difflib.unified_diff( + left_lines, right_lines, + fromfile=left_label, tofile=right_label, + )) + + return { + "vault": vault_name, + "path": path, + "version": version, + "compare_with": compare_with, + "diff": "".join(diff_lines), + "left_content": left_content, + "right_content": right_content, + } + except ServiceError: + raise + except Exception as e: + logger.error(f"Error generating diff for {vault_name}/{path}: {type(e).__name__}: {e}", exc_info=True) + raise ServiceError(f"Erreur lors de la génération du diff: {e!s}", code="read_error", status=500) from e diff --git a/backend/services/graph.py b/backend/services/graph.py new file mode 100644 index 0000000..6657d48 --- /dev/null +++ b/backend/services/graph.py @@ -0,0 +1,199 @@ +"""Graph data services shared by the REST route and the AI tool layer. + +Builds the nodes/edges structure consumed by the graph view. The route +``/api/graph/{vault}`` and the tool ``get_graph`` both delegate here so the +graph semantics (parent/child edges, wikilinks, tag filter) stay identical. +Permission checks are the caller's responsibility. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +from backend.indexer import SUPPORTED_EXTENSIONS, index +from backend.services.errors import ServiceError +from backend.services.paths import resolve_safe_path +from backend.services.vaults import get_vault_root + +_WIKILINK_PATTERN = re.compile(r"\[\[([^\]|#]+)(?:[|#][^\]]+)?\]\]") + + +def get_graph( + vault_name: str, + path: str = "", + depth: int = 1, + scope: str = "directory", + tag: str = "", +) -> dict[str, Any]: + """Return graph data (nodes and edges) for a vault or directory. + + Args: + vault_name: Name of the vault. + path: Relative directory path to focus on (empty = root). + depth: Expansion depth (0 = only direct children, 1-3 = deeper). + scope: ``"directory"`` for a subtree, ``"full"`` for the whole vault. + tag: Optional tag filter (only files with this tag appear). + + Returns: + Dict with ``vault``, ``path``, ``scope``, ``nodes`` and ``edges``. + + Raises: + ServiceError: ``not_found`` (404) when the vault or path is missing. + """ + from backend.vault_settings import get_vault_setting + + vault_root = get_vault_root(vault_name) + target = resolve_safe_path(vault_root, path) if path else vault_root.resolve() + + if not target.exists(): + raise ServiceError( + f"Path not found: {path}", + code="not_found", + status=404, + details={"vault": vault_name, "path": path}, + ) + + nodes: list[dict[str, Any]] = [] + edges: list[dict[str, Any]] = [] + node_ids: set[str] = set() + + def _add_node(name: str, ntype: str, npath: str, size: int = 0, + tags: list[str] | None = None, incoming: int = 0, outgoing: int = 0) -> str: + nid = f"{vault_name}:{npath}" + if nid not in node_ids: + node_ids.add(nid) + nodes.append({ + "id": nid, "name": name, "type": ntype, "path": npath, + "size": size, "tags": tags or [], + "incoming_count": incoming, "outgoing_count": outgoing, + }) + return nid + + def _add_edge(source: str, target_id: str, relation: str) -> None: + edges.append({"source": source, "target": target_id, "relation": relation}) + + settings = get_vault_setting(vault_name) or {} + hide_hidden = settings.get("hideHiddenFiles", False) + + # Build tag index from the in-memory index for fast lookups. + _tag_index: dict[str, list[str]] = {} + for doc_key, info in index.items(): + vn, fp = doc_key.split("::", 1) if "::" in doc_key else ("", "") + if vn == vault_name: + for t in info.get("tags", []): + _tag_index.setdefault(t.lower(), []).append(fp) + + if scope == "full": + target = vault_root.resolve() + effective_depth = depth if depth > 0 else 2 + else: + target = resolve_safe_path(vault_root, path) if path else vault_root.resolve() + effective_depth = depth + + if not target.exists(): + raise ServiceError( + f"Path not found: {path}", + code="not_found", + status=404, + details={"vault": vault_name, "path": path}, + ) + + focus_name = path.split("/")[-1] if path else vault_name + focus_type = "directory" if path else "vault" + focus_id = _add_node(focus_name, focus_type, path) + + def _walk_dir(dir_path: Path, parent_id: str, current_depth: int) -> None: + if current_depth > effective_depth: + return + try: + for entry in sorted(dir_path.iterdir(), key=lambda e: (not e.is_dir(), e.name.lower())): + if hide_hidden and entry.name.startswith("."): + continue + rel = str(entry.relative_to(vault_root)).replace("\\", "/") + + if tag and entry.is_file(): + file_tags = [t.lower() for t in _tag_index.get(rel, [])] + if tag.lower() not in file_tags: + continue + + if entry.is_dir(): + did = _add_node(entry.name, "directory", rel) + _add_edge(parent_id, did, "parent") + if current_depth < effective_depth: + _walk_dir(entry, did, current_depth + 1) + elif entry.suffix.lower() in SUPPORTED_EXTENSIONS or entry.name.lower() in ("dockerfile", "makefile"): + file_tags = _tag_index.get(rel, []) + fid = _add_node(entry.name, "file", rel, entry.stat().st_size, tags=file_tags) + _add_edge(parent_id, fid, "parent") + except PermissionError: + pass + + if target.is_dir(): + _walk_dir(target, focus_id, 0) + elif target.is_file(): + _walk_dir(target.parent, focus_id, 0) + + _add_wikilink_edges(nodes, edges, vault_name) + + edge_counts: dict[str, dict[str, int]] = {} + for node in nodes: + edge_counts[node["id"]] = {"incoming": 0, "outgoing": 0} + for edge in edges: + if edge["relation"] in ("wikilink", "backlink"): + src = edge["source"] + tgt = edge["target"] + if src in edge_counts: + edge_counts[src]["outgoing"] += 1 + if tgt in edge_counts: + edge_counts[tgt]["incoming"] += 1 + for node in nodes: + counts = edge_counts.get(node["id"], {"incoming": 0, "outgoing": 0}) + node["incoming_count"] = counts["incoming"] + node["outgoing_count"] = counts["outgoing"] + + return {"vault": vault_name, "path": path, "scope": scope, "nodes": nodes, "edges": edges} + + +def _add_wikilink_edges(nodes: list[dict[str, Any]], edges: list[dict[str, Any]], vault_name: str) -> None: + """Add edges for wikilinks between markdown files in the current graph scope.""" + file_nodes = [n for n in nodes if n["type"] == "file" and n["path"].endswith(".md")] + if len(file_nodes) < 2: + return + + path_to_id = {n["path"]: n["id"] for n in file_nodes} + existing = {(e["source"], e["target"]) for e in edges} + + for node in file_nodes: + vault_data = index.get(vault_name) + if not vault_data: + continue + file_entry = None + for f in vault_data.get("files", []): + if f["path"] == node["path"]: + file_entry = f + break + if not file_entry: + continue + + content = file_entry.get("content", "") + if not content: + continue + + for match in _WIKILINK_PATTERN.finditer(content): + target = match.group(1).strip() + target_lower = target.lower() + if not target_lower.endswith(".md"): + target_lower += ".md" + + for target_path, target_id in path_to_id.items(): + if target_id == node["id"]: + continue + target_name = target_path.rsplit("/", 1)[-1].lower() + if target_name == target_lower or target_path.lower() == target_lower: + edge_key = tuple(sorted([node["id"], target_id])) + if edge_key not in existing and (edge_key[1], edge_key[0]) not in existing: + edges.append({"source": node["id"], "target": target_id, "relation": "wikilink"}) + existing.add((node["id"], target_id)) + break diff --git a/backend/services/recent.py b/backend/services/recent.py new file mode 100644 index 0000000..4d95a43 --- /dev/null +++ b/backend/services/recent.py @@ -0,0 +1,138 @@ +"""Recent-files services shared by the REST route and the AI tool layer. + +Provides ``list_recent`` (last opened files, falling back to last modified) +and ``humanize_mtime``. The route ``/api/recent`` and the tool ``list_recent`` +both delegate here. Permission filtering is applied against the caller's +vault list. +""" + +from __future__ import annotations + +import time +from datetime import datetime +from typing import Any + +from backend.history import get_recent_opened, is_bookmarked +from backend.indexer import find_file_in_index, index + + +def humanize_mtime(mtime: float) -> str: + """Return a short, human-friendly French rendering of a timestamp.""" + delta = time.time() - mtime + if delta < 60: + return "à l'instant" + if delta < 3600: + return f"il y a {int(delta / 60)} min" + if delta < 86400: + return f"il y a {int(delta / 3600)} h" + if delta < 604800: + return f"il y a {int(delta / 86400)} j" + return datetime.fromtimestamp(mtime).strftime("%d %b %Y") + + +def _can_access(vault: str, user_vaults: list[str]) -> bool: + return "*" in user_vaults or vault in user_vaults + + +def list_recent( + username: str | None, + user_vaults: list[str] | None = None, + *, + vault: str | None = None, + limit: int = 20, + mode: str = "opened", +) -> dict[str, Any]: + """Return the caller's recent files. + + Args: + username: Caller username (needed for the "opened" history mode). + user_vaults: Vaults the caller may access (``["*"]`` = all). + vault: Optional single-vault filter. + limit: Maximum number of files to return. + mode: ``"opened"`` to use the open history, anything else for the + last-modified fallback. + + Returns: + Dict with ``files``, ``total``, ``limit`` and ``mode``. + """ + user_vaults = user_vaults or [] + + if mode == "opened" and username: + history = get_recent_opened(username, vault_filter=vault, limit=limit) + files_resp: list[dict[str, Any]] = [] + for item in history: + v_name = item["vault"] + if not _can_access(v_name, user_vaults): + continue + + f_idx = find_file_in_index(item["path"], v_name) + if f_idx: + files_resp.append({ + "path": f_idx["path"], + "title": f_idx.get("title") or item["path"].split("/")[-1], + "vault": v_name, + "mtime": item["opened_at"], + "mtime_human": humanize_mtime(item["opened_at"]), + "size_bytes": f_idx.get("size", 0), + "tags": [f"#{t}" for t in f_idx.get("tags", [])][:5], + "preview": f_idx.get("content_preview", "")[:120], + "bookmarked": is_bookmarked(username, v_name, f_idx["path"]), + }) + else: + files_resp.append({ + "path": item["path"], + "title": item.get("title") or item["path"].split("/")[-1], + "vault": v_name, + "mtime": item["opened_at"], + "mtime_human": humanize_mtime(item["opened_at"]), + "tags": [], + "preview": "", + "bookmarked": is_bookmarked(username, v_name, item["path"]), + }) + return { + "files": files_resp, + "total": len(files_resp), + "limit": limit, + "mode": "opened", + } + + all_files: list[tuple[str, dict[str, Any]]] = [] + for v_name, v_data in index.items(): + if vault and v_name != vault: + continue + if not _can_access(v_name, user_vaults): + continue + for f in v_data.get("files", []): + all_files.append((v_name, f)) + + all_files.sort(key=lambda x: x[1].get("modified", ""), reverse=True) + recent = all_files[:limit] + + files_resp = [] + for v_name, f in recent: + iso_modified = f.get("modified", "") + try: + mtime_dt = datetime.fromisoformat(iso_modified.replace("Z", "+00:00")) + mtime_val = mtime_dt.timestamp() + except Exception: + mtime_val = time.time() + + files_resp.append({ + "path": f["path"], + "title": f["title"], + "vault": v_name, + "mtime": mtime_val, + "mtime_human": humanize_mtime(mtime_val), + "mtime_iso": iso_modified, + "size_bytes": f.get("size", 0), + "tags": [f"#{t}" for t in f.get("tags", [])][:5], + "preview": f.get("content_preview", "")[:120], + "bookmarked": is_bookmarked(username, v_name, f["path"]) if username else False, + }) + + return { + "files": files_resp, + "total": len(all_files), + "limit": limit, + "mode": "modified", + } diff --git a/backend/services/search.py b/backend/services/search.py index fc09805..6be7f18 100644 --- a/backend/services/search.py +++ b/backend/services/search.py @@ -39,3 +39,74 @@ def list_tags(vault: str | None = None) -> dict[str, int]: from backend.search import get_all_tags return get_all_tags(vault_filter=vault) + + +def advanced_search_vaults( + query: str = "", + vault: str = "all", + tag: str | None = None, + limit: int = 50, + offset: int = 0, + sort: str = "relevance", + case_sensitive: bool = False, + whole_word: bool = False, + regex: bool = False, + include_paths: str | None = None, + exclude_paths: str | None = None, + created: str | None = None, + modified: str | None = None, + size: str | None = None, +) -> dict[str, Any]: + """Advanced full-text search (TF-IDF, facets, operators). + + No permission filtering is applied: callers that need it (the tool layer) + filter the ``results`` list themselves. + """ + from backend.search import advanced_search + + return advanced_search( + query, + vault_filter=vault, + tag_filter=tag, + limit=limit, + offset=offset, + sort_by=sort, + case_sensitive=case_sensitive, + whole_word=whole_word, + regex=regex, + include_paths=include_paths, + exclude_paths=exclude_paths, + created=created, + modified=modified, + size=size, + ) + + +def search_paths(q: str, vault: str = "all") -> dict[str, Any]: + """Search files and directories by path substring using the path index. + + No permission filtering is applied: callers that need it (the tool layer) + filter the ``results`` list themselves. + """ + from backend.indexer import path_index + + if not q: + return {"query": q, "vault_filter": vault, "results": []} + + query_lower = q.lower() + results: list[dict[str, Any]] = [] + + vaults_to_search = [vault] if vault != "all" else list(path_index.keys()) + + for vault_name in vaults_to_search: + for entry in path_index.get(vault_name, []): + if query_lower in entry["name"].lower() or query_lower in entry["path"].lower(): + results.append({ + "vault": vault_name, + "path": entry["path"], + "name": entry["name"], + "type": entry["type"], + "matched_path": entry["path"], + }) + + return {"query": q, "vault_filter": vault, "results": results} diff --git a/backend/services/vaults.py b/backend/services/vaults.py index dd802a3..2e3ad44 100644 --- a/backend/services/vaults.py +++ b/backend/services/vaults.py @@ -100,3 +100,115 @@ def browse_directory(vault_name: str, path: str = "") -> dict[str, Any]: raise ServiceError("Permission denied", code="permission_denied", status=403) from None return {"vault": vault_name, "path": path, "items": items} + + +def list_all_files( + vault_name: str, + dir: str = "", + limit: int = 200, + recursive: bool = True, +) -> dict[str, Any]: + """List files in a vault directory sorted by modification time (newest first). + + Unlike :func:`browse_directory`, this returns files only (with metadata: + size, mtime, extension) and can recurse into subdirectories. Hidden files + and ignored directories are skipped. + + Args: + vault_name: Name of the vault. + dir: Relative directory path within the vault (empty = root). + limit: Maximum number of files to return. + recursive: Recurse into subdirectories when True. + + Returns: + Dict with ``vault``, ``directory``, ``recursive``, ``count`` and + ``files``. + + Raises: + ServiceError: ``not_found`` (404) if the directory does not exist, + ``permission_denied`` (403) if it cannot be read. + """ + from datetime import datetime, timezone + + from backend.indexer import IGNORED_DIRS + + root = get_vault_root(vault_name) + dir_path = resolve_safe_path(root, dir) if dir else root + + if not dir_path.exists() or not dir_path.is_dir(): + raise ServiceError( + f"Directory not found: {dir}", + code="not_found", + status=404, + details={"vault": vault_name, "path": dir}, + ) + + files: list[dict[str, Any]] = [] + dir_prefix = (dir or "").strip("/") + + try: + iterator = dir_path.rglob("*") if recursive else dir_path.iterdir() + for entry in iterator: + if not entry.is_file(): + continue + if entry.name.startswith("."): + continue + if entry.name in IGNORED_DIRS: + continue + + if recursive and dir_prefix: + skip = False + try: + for part in entry.relative_to(dir_path).parts[:-1]: + if part.startswith(".") or part in IGNORED_DIRS: + skip = True + break + except ValueError: + pass + if skip: + continue + + stat = entry.stat() + rel_path = str(entry.relative_to(root)).replace("\\", "/") + + if recursive and dir_prefix: + try: + rel_to_dir = str(entry.parent.relative_to(dir_path)).replace("\\", "/") + except ValueError: + rel_to_dir = "" + else: + rel_to_dir = "" + + ext = entry.suffix.lower() if entry.suffix else "" + + file_entry = { + "name": entry.name, + "path": rel_path, + "vault": vault_name, + "size": stat.st_size, + "modified": stat.st_mtime, + "modified_iso": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(), + "extension": ext.lstrip(".") if ext else "", + } + if rel_to_dir and rel_to_dir != ".": + file_entry["rel_dir"] = rel_to_dir + + files.append(file_entry) + except PermissionError: + raise ServiceError( + "Permission denied reading directory", + code="permission_denied", + status=403, + details={"vault": vault_name, "path": dir}, + ) from None + + files.sort(key=lambda f: f["modified"], reverse=True) + files = files[:limit] + + return { + "vault": vault_name, + "directory": dir, + "recursive": recursive, + "count": len(files), + "files": files, + } diff --git a/backend/tools/schemas.py b/backend/tools/schemas.py index 8f295d9..ae3c345 100644 --- a/backend/tools/schemas.py +++ b/backend/tools/schemas.py @@ -44,6 +44,100 @@ class ListTagsInput(BaseModel): vault: str | None = Field(None, description="Vault name or 'all' (default: all accessible)") +class ListAllFilesInput(BaseModel): + """List every file of a vault (optionally under a subdirectory).""" + + vault: str = Field(..., description="Vault name") + dir: str = Field("", description="Vault-relative directory path (empty = vault root)") + limit: int = Field(200, ge=1, le=2000, description="Maximum number of files") + recursive: bool = Field(True, description="Recurse into subdirectories") + + +class ReadFileRawInput(BaseModel): + """Read a text file's raw content (no HTML rendering).""" + + vault: str = Field(..., description="Vault name") + path: str = Field(..., description="Vault-relative file path") + + +class GetBacklinksInput(BaseModel): + """List files linking to a target file via wikilinks.""" + + vault: str = Field(..., description="Vault name") + path: str = Field(..., description="Vault-relative path of the target file") + + +class ListBackupsInput(BaseModel): + """List available backup versions of a file.""" + + vault: str = Field(..., description="Vault name") + path: str = Field(..., description="Vault-relative file path") + + +class DiffBackupInput(BaseModel): + """Unified diff between a backup version and another version or the current file.""" + + vault: str = Field(..., description="Vault name") + path: str = Field(..., description="Vault-relative file path") + version: int = Field(..., description="Backup timestamp used as the old/left side") + compare_with: int | None = Field( + None, + description="Backup timestamp used as the new/right side (omit to compare with the current file)", + ) + + +class GetGraphInput(BaseModel): + """Graph data (nodes and edges) for a vault or directory.""" + + vault: str = Field(..., description="Vault name") + path: str = Field("", description="Vault-relative directory path to focus on (empty = root)") + depth: int = Field(1, ge=0, le=3, description="Expansion depth") + scope: str = Field("directory", description="'directory' for a subtree, 'full' for the whole vault") + tag: str = Field("", description="Only include files carrying this tag") + + +class SearchAdvancedInput(BaseModel): + """Advanced full-text search (TF-IDF, facets, operators).""" + + q: str = Field("", description="Query (supports tag:, vault:, title:, path:, ext: operators)") + vault: str = Field("all", description="Vault name or 'all'") + tag: str | None = Field(None, description="Optional comma-separated tag filter") + limit: int = Field(50, ge=1, le=200, description="Maximum number of results") + offset: int = Field(0, ge=0, description="Pagination offset") + sort: str = Field("relevance", description="Sort by 'relevance' or 'modified'") + case_sensitive: bool = Field(False, description="Match case") + whole_word: bool = Field(False, description="Match whole words only") + regex: bool = Field(False, description="Treat query as a regular expression") + include_paths: str | None = Field(None, description="Comma-separated glob patterns to include") + exclude_paths: str | None = Field(None, description="Comma-separated glob patterns to exclude") + created: str | None = Field(None, description="Created date filter (>date, list[dict[st ] +@tool( + name="list_all_files", + description="List every file of a vault (optionally under a subdirectory), newest first.", + input_model=ListAllFilesInput, + risk=ToolRisk.READ, + requires_vault=True, +) +def list_all_files_tool(ctx: ToolContext, params: ListAllFilesInput) -> dict[str, Any]: + """Return a flat list of files with metadata.""" + return list_all_files(params.vault, dir=params.dir, limit=params.limit, recursive=params.recursive) + + +# ── C2. Content reading ──────────────────────────────────────────────────── + + @tool( name="read_file", description="Read the text content of a file inside a vault (secrets redacted).", @@ -78,6 +113,81 @@ def read_file(ctx: ToolContext, params: ReadFileInput) -> dict[str, Any]: ) +@tool( + name="read_file_raw", + description="Read a file's raw text content, without the size cap (secrets redacted).", + input_model=ReadFileRawInput, + risk=ToolRisk.READ, + requires_vault=True, +) +def read_file_raw(ctx: ToolContext, params: ReadFileRawInput) -> dict[str, Any]: + """Return the full (redacted) raw text of a vault file.""" + data = read_file_text(params.vault, params.path, redact=True, max_bytes=None) + return {"vault": data["vault"], "path": data["path"], "raw": data["content"]} + + +@tool( + name="get_backlinks", + description="List files that link to a target file via [[wikilinks]].", + input_model=GetBacklinksInput, + risk=ToolRisk.READ, + requires_vault=True, +) +def get_backlinks(ctx: ToolContext, params: GetBacklinksInput) -> list[dict[str, Any]]: + """Return backlinks, filtered to accessible vaults.""" + backlinks = _get_backlinks(params.vault, params.path) + return [b for b in backlinks if ctx.has_vault_access(b["vault"])] + + +@tool( + name="list_backups", + description="List the available backup versions of a file (newest first).", + input_model=ListBackupsInput, + risk=ToolRisk.READ, + requires_vault=True, +) +def list_backups(ctx: ToolContext, params: ListBackupsInput) -> dict[str, Any]: + """Return the backup versions of a vault file.""" + return { + "vault": params.vault, + "path": params.path, + "backups": _list_backup_files(params.vault, params.path), + } + + +@tool( + name="diff_backup", + description="Show a unified diff between a backup version and another version or the current file.", + input_model=DiffBackupInput, + risk=ToolRisk.READ, + requires_vault=True, +) +def diff_backup(ctx: ToolContext, params: DiffBackupInput) -> dict[str, Any]: + """Return the unified diff for a backup version.""" + return _diff_backup(params.vault, params.path, params.version, params.compare_with) + + +@tool( + name="get_graph", + description="Return the graph (nodes and edges: parent/child + wikilinks) of a vault or directory.", + input_model=GetGraphInput, + risk=ToolRisk.READ, + requires_vault=True, +) +def get_graph(ctx: ToolContext, params: GetGraphInput) -> dict[str, Any]: + """Return graph data for a vault subtree or the whole vault.""" + return _get_graph( + params.vault, + path=params.path, + depth=params.depth, + scope=params.scope, + tag=params.tag, + ) + + +# ── C3. Search ───────────────────────────────────────────────────────────── + + @tool( name="search_fulltext", description="Full-text search across one vault or all accessible vaults.", @@ -90,6 +200,45 @@ def search_fulltext(ctx: ToolContext, params: SearchFulltextInput) -> list[dict[ return [r for r in payload["results"] if ctx.has_vault_access(r["vault"])] +@tool( + name="search_advanced", + description="Advanced full-text search with operators, filters and facets.", + input_model=SearchAdvancedInput, + risk=ToolRisk.READ, +) +def search_advanced(ctx: ToolContext, params: SearchAdvancedInput) -> list[dict[str, Any]]: + """Return advanced search results, filtered to accessible vaults.""" + payload = advanced_search_vaults( + params.q, + vault=params.vault, + tag=params.tag, + limit=params.limit, + offset=params.offset, + sort=params.sort, + case_sensitive=params.case_sensitive, + whole_word=params.whole_word, + regex=params.regex, + include_paths=params.include_paths, + exclude_paths=params.exclude_paths, + created=params.created, + modified=params.modified, + size=params.size, + ) + return [r for r in payload["results"] if ctx.has_vault_access(r["vault"])] + + +@tool( + name="search_paths", + description="Search files and directories by path substring.", + input_model=SearchPathsInput, + risk=ToolRisk.READ, +) +def search_paths(ctx: ToolContext, params: SearchPathsInput) -> list[dict[str, Any]]: + """Return path matches, filtered to accessible vaults.""" + payload = _search_paths(params.q, vault=params.vault) + return [r for r in payload["results"] if ctx.has_vault_access(r["vault"])] + + @tool( name="list_tags", description="List tags with their occurrence counts for a vault or all accessible vaults.", @@ -102,8 +251,6 @@ def list_tags(ctx: ToolContext, params: ListTagsInput) -> list[dict[str, Any]]: ctx.require_vault_access(params.vault) return [{"tag": tag, "count": count} for tag, count in _list_tags(params.vault).items()] - from backend.indexer import get_vault_names - merged: dict[str, int] = {} for name in get_vault_names(): if not ctx.has_vault_access(name): @@ -111,3 +258,45 @@ def list_tags(ctx: ToolContext, params: ListTagsInput) -> list[dict[str, Any]]: for tag, count in _list_tags(name).items(): merged[tag] = merged.get(tag, 0) + count return [{"tag": tag, "count": count} for tag, count in sorted(merged.items(), key=lambda x: -x[1])] + + +@tool( + name="suggest_tags", + description="Suggest tags matching a prefix, across accessible vaults.", + input_model=SuggestTagsInput, + risk=ToolRisk.READ, +) +def suggest_tags(ctx: ToolContext, params: SuggestTagsInput) -> list[dict[str, Any]]: + """Return tag suggestions, restricted to accessible vaults.""" + from backend.search import suggest_tags as _suggest + + if params.vault and params.vault != "all": + ctx.require_vault_access(params.vault) + return _suggest(params.q, vault_filter=params.vault, limit=params.limit) + + merged: dict[str, int] = {} + for name in get_vault_names(): + if not ctx.has_vault_access(name): + continue + for item in _suggest(params.q, vault_filter=name, limit=params.limit): + merged[item["tag"]] = merged.get(item["tag"], 0) + item["count"] + ordered = sorted(merged.items(), key=lambda x: -x[1]) + return [{"tag": tag, "count": count} for tag, count in ordered[: params.limit]] + + +@tool( + name="list_recent", + description="List the current user's recently opened (or modified) files.", + input_model=ListRecentInput, + risk=ToolRisk.READ, +) +def list_recent(ctx: ToolContext, params: ListRecentInput) -> dict[str, Any]: + """Return recent files for the calling user.""" + user_vaults = ctx.user.get("_token_vaults") or ctx.user.get("vaults", []) + return _list_recent( + ctx.username, + user_vaults, + vault=params.vault, + limit=params.limit, + mode=params.mode, + ) diff --git a/docs/AI_ARCHITECTURE_GUIDE.md b/docs/AI_ARCHITECTURE_GUIDE.md index 0bcb237..159fe2c 100644 --- a/docs/AI_ARCHITECTURE_GUIDE.md +++ b/docs/AI_ARCHITECTURE_GUIDE.md @@ -93,21 +93,25 @@ ObsiGate possède aujourd'hui **deux sous-systèmes IA** qui partagent la même Extraire la logique métier des routes de `backend/main.py` vers des fonctions réutilisables (services). Les routes REST, l'agent in-app et le serveur MCP appellent ces mêmes services. -**Nouveau module `backend/services/` (A2 — lecture/recherche) :** +**Nouveau module `backend/services/` (A2 lecture/recherche + C) :** ``` backend/services/ ├── errors.py # ServiceError (code + status HTTP) ├── paths.py # resolve_safe_path (anti path-traversal, source unique) -├── vaults.py # list_accessible_vaults, browse_directory, get_vault_root +├── vaults.py # list_accessible_vaults, browse_directory, get_vault_root, list_all_files ├── files.py # read_raw_file, read_file_text (redaction + quota) -└── search.py # search_vaults (pagination), list_tags +├── search.py # search_vaults (pagination), list_tags, advanced_search_vaults, search_paths +├── backups.py # get_backup_dir, list_backup_files, diff_backup +├── graph.py # get_graph (nodes/edges, wikilinks) +└── recent.py # list_recent, humanize_mtime ``` -Les routes `/api/vaults`, `/api/browse/{vault}`, `/api/file/{vault}/raw`, `/api/search` et -`/api/tags` en sont de simples wrappers, tout comme les outils `list_vaults`, `list_directory`, -`read_file`, `search_fulltext` et `list_tags`. Un `ServiceError` est traduit en `HTTPException` -(handler global de `backend/main.py`) ou en `ToolError` (`backend/tools/registry.py`). +Les routes `/api/vaults`, `/api/browse/{vault}`, `/api/file/{vault}/raw`, `/api/search`, `/api/tags`, +`/api/recent`, `/api/vault/{vault}/files`, `/api/file/{vault}/backups`, `/api/file/{vault}/diff`, +`/api/graph/{vault}`, `/api/tree-search` et `/api/search/advanced` en sont de simples wrappers, tout +comme les outils correspondants du catalogue (phase C). Un `ServiceError` est traduit en +`HTTPException` (handler global de `backend/main.py`) ou en `ToolError` (`backend/tools/registry.py`). **Nouveau module `backend/tools/` :** @@ -306,7 +310,7 @@ Voir `docs/ROADMAP.md` (item dédié) pour le détail des activités. ## 9. Références - `backend/ai.py`, `backend/ai_routes.py` — couche fournisseurs + actions éditeur -- `backend/services/` — logique métier partagée (vaults, files, search) consommée par les routes et les outils +- `backend/services/` — logique métier partagée (vaults, files, search, backups, graph, recent) consommée par les routes et les outils - `backend/ai_chat.py` — chat completion provider-agnostique avec tool calling et streaming (OpenAI-compat + Gemini) - `backend/agent/loop.py` — agent loop in-app (multi-étapes, LLM injectable) - `backend/bookslm.py`, `backend/bookslm_routes.py` — assistant contextuel (+ endpoint `/agent`) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d211f38..a266bce 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -43,10 +43,10 @@ ### 79. Assistant IA — Outils (function calling) & serveur MCP - **Effort :** 10-15 jours | **Impact :** 🟡 -- **Statut :** 🔵 Phase 0 + A2 + B1/B2/B3/B4/B5/B6/B7 + G livrés (2026-09-11). Détail complet : [features/ai-tools-mcp.md](./features/ai-tools-mcp.md) +- **Statut :** 🔵 Phase 0 + A2 + B1/B2/B3/B4/B5/B6/B7 + C + G livrés (2026-09-11). Détail complet : [features/ai-tools-mcp.md](./features/ai-tools-mcp.md) - **Description :** Transformer l'assistant BooksLM en agent (lire, chercher, lister, ouvrir, modifier) via **function calling natif**, puis exposer ObsiGate à des **clients MCP externes** (Claude Desktop, Cursor…). Les deux fronts consomment une **couche d'outils partagée**. - **Reste à faire :** - - [ ] **C.** Catalogue lecture & recherche (vaults, read, backlinks, backups, search, tags) + - [x] **C.** Catalogue lecture & recherche (vaults, read, backlinks, backups, search, tags) - [ ] **D.** Catalogue mutations (create/edit/append/rename/move/delete) + confirmations two-step - [ ] **E.** Serveur MCP (Streamable HTTP `/mcp`, auth Bearer JWT, `propose`/`apply`, toggle par vault) - [ ] **F.** Durcissement (rate limiting, redaction secrets, OpenAPI + guide MCP, E2E) @@ -180,7 +180,7 @@ |---|---|---| | ✅ Complété | #1 → #59, #61, #63–68, #71, #72, #74–76, #78 | ~82 jours réalisés | | 🔵 P2 restant | #77 Desktop : signature code (optionnel), wizard 1er lancement (optionnel), 6 tests E2E **manuels** | ~1-2 jours | -| ⚪ P3 restant | #62 Collaboration Yjs (5-7j) · #79 Assistant IA outils + MCP (10-15j, Phase 0 + A2 + B + G livrés) · #80 Assistant IA UX (livré, en attente vérif) | ~15-22 jours | +| ⚪ P3 restant | #62 Collaboration Yjs (5-7j) · #79 Assistant IA outils + MCP (10-15j, Phase 0 + A2 + B + C + G livrés) · #80 Assistant IA UX (livré, en attente vérif) | ~15-22 jours | | ⚪ P4 restant | #69 Mobile éditeur (2-3j) · #70 Sémantique (4-5j) · #73 Sync (6-8j) | 11-16 jours | | **Total restant** | **6 items + finitions** | **~27-39 jours** | diff --git a/docs/features/ai-tools-mcp.md b/docs/features/ai-tools-mcp.md index 268492a..451aa53 100644 --- a/docs/features/ai-tools-mcp.md +++ b/docs/features/ai-tools-mcp.md @@ -1,6 +1,6 @@ # #79 — Assistant IA — Outils (function calling) & serveur MCP -> **Statut :** 🔵 En cours — Phase 0 + A2 + B1/B2/B3/B4/B5/B6/B7 + G livrés (2026-09-11) +> **Statut :** 🔵 En cours — Phase 0 + A2 + B1/B2/B3/B4/B5/B6/B7 + C + G livrés (2026-09-11) > **Effort :** 10-15 jours | **Impact :** 🟡 > **Références :** [Roadmap](../ROADMAP.md) · [Guide d'architecture IA](../AI_ARCHITECTURE_GUIDE.md) @@ -23,10 +23,11 @@ - [x] **B6.** Outils de navigation in-app : `open_file`, `reveal_in_tree` (événement `obsigate:open-file`) — livré via les liens cliquables de l'assistant (#80, [ai-assistant-ux.md](./ai-assistant-ux.md)) - [x] **B7.** Tests : agent loop LLM mocké (`tests/test_agent_loop.py`), providers (`tests/test_ai_chat.py`), endpoint (`tests/test_bookslm.py`) -## C. Catalogue d'outils — lecture & recherche (1-2 jours) - - [ ] **C1.** Vaults/navigation : `list_vaults`, `list_directory`, `list_all_files` - - [ ] **C2.** Lecture : `read_file`, `read_file_raw`, `get_backlinks`, `list_backups`, `diff_backup`, `get_graph` - - [ ] **C3.** Recherche : `search_fulltext`, `search_advanced`, `search_paths`, `list_tags`, `suggest_tags`, `list_recent` +## C. Catalogue d'outils — lecture & recherche (1-2 jours) — ✅ livré (2026-09-11) + - [x] **C1.** Vaults/navigation : `list_vaults`, `list_directory`, `list_all_files` — `list_all_files` ajouté via `backend/services/vaults.py` + - [x] **C2.** Lecture : `read_file`, `read_file_raw`, `get_backlinks`, `list_backups`, `diff_backup`, `get_graph` — services `backend/services/backups.py` (backups/diff) et `graph.py` ; `read_file_raw` redacte les secrets + - [x] **C3.** Recherche : `search_fulltext`, `search_advanced`, `search_paths`, `list_tags`, `suggest_tags`, `list_recent` — extensions `backend/services/search.py` (`advanced_search_vaults`, `search_paths`) et `recent.py` ; filtrage systématique par permissions vault + - Routes `/api/recent`, `/api/vault/{vault}/files`, `/api/file/{vault}/backups`, `/api/file/{vault}/diff`, `/api/graph/{vault}`, `/api/tree-search`, `/api/search/advanced` refactorées en wrappers des services ; tests `tests/test_tools.py` (+17) ## D. Catalogue d'outils — mutations (2 jours) - [ ] **D1.** `create_file`, `create_directory` (migration du protocole texte existant) diff --git a/tests/test_tools.py b/tests/test_tools.py index 2debc83..76287a3 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -28,7 +28,25 @@ from backend.tools.api import ( from backend.tools.registry import ToolSpec from backend.tools.schemas import ListVaultsInput -BUILTIN_TOOLS = {"list_vaults", "list_directory", "read_file", "search_fulltext", "list_tags"} +BUILTIN_TOOLS = { + # Phase 0 + "list_vaults", + "list_directory", + "read_file", + "search_fulltext", + "list_tags", + # Phase C — read/search catalogue + "list_all_files", + "read_file_raw", + "get_backlinks", + "list_backups", + "diff_backup", + "get_graph", + "search_advanced", + "search_paths", + "suggest_tags", + "list_recent", +} def _ctx(vaults=None, **kwargs) -> ToolContext: @@ -278,3 +296,137 @@ class TestToolAudit: assert safe["vault"] == "V" assert safe["path"] == "a.md" assert safe["content"] == "<500 chars>" + + +# ═══════════════════════════════════════════════════════════════════ +# Phase C — read/search catalogue +# ═══════════════════════════════════════════════════════════════════ + + +def _vault_path() -> Path: + from backend.indexer import get_vault_data + + return Path(get_vault_data("TestVault")["path"]) + + +def _make_backup(rel_path: str, ts: int, content: str) -> Path: + from backend.services.backups import get_backup_dir + + backup_dir = get_backup_dir("TestVault", rel_path) + backup_dir.mkdir(parents=True, exist_ok=True) + backup = backup_dir / f"{Path(rel_path).name}.{ts}.bak" + backup.write_text(content, encoding="utf-8") + return backup + + +class TestPhaseCTools: + def test_list_all_files(self, client): + result = call_tool("list_all_files", _ctx(), {"vault": "TestVault"}) + assert result.ok + paths = {f["path"] for f in result.data["files"]} + assert "note1.md" in paths + assert "Projets/projet.md" in paths + + def test_list_all_files_non_recursive(self, client): + result = call_tool( + "list_all_files", _ctx(), {"vault": "TestVault", "recursive": False} + ) + paths = {f["path"] for f in result.data["files"]} + assert "note1.md" in paths + assert "Projets/projet.md" not in paths + + def test_list_all_files_permission_denied(self, client): + ctx = ToolContext(user={"username": "limited", "vaults": ["OtherVault"]}, audit_enabled=False) + with pytest.raises(ToolPermissionError): + call_tool("list_all_files", ctx, {"vault": "TestVault"}) + + def test_read_file_raw(self, client): + result = call_tool("read_file_raw", _ctx(), {"vault": "TestVault", "path": "note1.md"}) + assert result.ok + assert "Python" in result.data["raw"] + assert result.data["path"] == "note1.md" + + def test_get_backlinks(self, client): + result = call_tool("get_backlinks", _ctx(), {"vault": "TestVault", "path": "note1.md"}) + assert result.ok + assert all(b["vault"] == "TestVault" for b in result.data) + + def test_get_backlinks_filters_inaccessible_vaults(self, client): + ctx = ToolContext(user={"username": "limited", "vaults": ["OtherVault"]}, audit_enabled=False) + with pytest.raises(ToolPermissionError): + call_tool("get_backlinks", ctx, {"vault": "TestVault", "path": "note1.md"}) + + def test_list_backups_empty(self, client): + result = call_tool("list_backups", _ctx(), {"vault": "TestVault", "path": "note1.md"}) + assert result.ok + assert result.data["backups"] == [] + + def test_list_backups_and_diff(self, client): + backup = _make_backup("note1.md", 1_700_000_000, "old content\n") + try: + listed = call_tool("list_backups", _ctx(), {"vault": "TestVault", "path": "note1.md"}) + timestamps = [b["timestamp"] for b in listed.data["backups"]] + assert 1_700_000_000 in timestamps + + diff = call_tool( + "diff_backup", + _ctx(), + {"vault": "TestVault", "path": "note1.md", "version": 1_700_000_000}, + ) + assert diff.ok + assert diff.data["version"] == 1_700_000_000 + assert diff.data["left_content"] == "old content\n" + assert "-old content" in diff.data["diff"] + finally: + backup.unlink(missing_ok=True) + + def test_diff_backup_missing_version(self, client): + with pytest.raises(ToolNotFoundError): + call_tool( + "diff_backup", + _ctx(), + {"vault": "TestVault", "path": "note1.md", "version": 42}, + ) + + def test_get_graph(self, client): + result = call_tool("get_graph", _ctx(), {"vault": "TestVault", "scope": "full", "depth": 2}) + assert result.ok + node_paths = {n["path"] for n in result.data["nodes"]} + assert "note1.md" in node_paths + assert result.data["scope"] == "full" + + def test_get_graph_missing_path(self, client): + with pytest.raises(ToolNotFoundError): + call_tool("get_graph", _ctx(), {"vault": "TestVault", "path": "nope"}) + + def test_search_advanced(self, client): + result = call_tool("search_advanced", _ctx(), {"q": "python"}) + assert result.ok + assert len(result.data) >= 1 + assert all(r["vault"] == "TestVault" for r in result.data) + + def test_search_paths(self, client): + result = call_tool("search_paths", _ctx(), {"q": "note1"}) + assert result.ok + assert any(r["path"] == "note1.md" for r in result.data) + + def test_suggest_tags(self, client): + result = call_tool("suggest_tags", _ctx(), {"q": "py"}) + assert result.ok + assert any(s["tag"] == "python" for s in result.data) + + def test_suggest_tags_specific_vault_denied(self, client): + ctx = ToolContext(user={"username": "limited", "vaults": ["OtherVault"]}, audit_enabled=False) + with pytest.raises(ToolPermissionError): + call_tool("suggest_tags", ctx, {"q": "py", "vault": "TestVault"}) + + def test_list_recent_modified(self, client): + result = call_tool("list_recent", _ctx(), {"mode": "modified"}) + assert result.ok + assert result.data["mode"] == "modified" + assert any(f["path"] == "note1.md" for f in result.data["files"]) + + def test_list_recent_filters_by_vault(self, client): + ctx = ToolContext(user={"username": "limited", "vaults": ["OtherVault"]}, audit_enabled=False) + result = call_tool("list_recent", ctx, {"mode": "modified"}) + assert result.data["files"] == []