ObsiGate/backend/tools/spreadsheets.py

"""Spreadsheet tools (#153 A6, #156 A14) — read and mutate existing workbooks.

Complements :mod:`backend.tools.documents` (``create_xlsx`` creates a *new*
file; here the assistant can read and edit one that already exists). The same
three formats the viewer edits are supported — ``.xlsx``, ``.xlsm`` and
``.csv`` (#156-A14 used to be ``.xlsx`` only, which made a workbook the UI
edits invisible to the assistant):

* ``list_xlsx_sheets``    — READ, sheet names + dimensions;
* ``xlsx_to_markdown``    — READ, bounded markdown table for the LLM context;
* ``search_workbook``     — READ, find text across every sheet (#156-A14);
* ``analyze_range``       — READ, aggregate stats over an A1 range (#156-A14);
* ``update_xlsx_cells``   — WRITE, batch cell edits (guarded service);
* ``append_xlsx_rows``    — WRITE, append whole rows at the end of a sheet;
* ``edit_xlsx_structure`` — WRITE, structural CRUD (sheets/rows/columns).

Mutation tools go through :func:`backend.services.mutations.edit_xlsx_cells`
(or ``save_csv_cells`` / ``mutate_xlsx_structure``), which already carry the
#153 P0 guards: per-file lock, atomic replace, formula neutralisation
(``allow_formula`` opt-in) and the lossy-write 409. Every write is a WRITE-risk
tool, so the registry keeps asking for an explicit confirmation.
"""

from __future__ import annotations

import logging
import re
from pathlib import Path
from typing import Any

from backend.services.errors import ServiceError
from backend.services.paths import resolve_safe_path
from backend.services.vaults import get_vault_root
from backend.tools.context import ToolContext, ToolError, ToolRisk
from backend.tools.registry import tool
from backend.tools.schemas import (
    AnalyzeRangeInput,
    AppendXlsxRowsInput,
    EditXlsxStructureInput,
    ListXlsxSheetsInput,
    SearchWorkbookInput,
    UpdateXlsxCellsInput,
    XlsxToMarkdownInput,
)

logger = logging.getLogger("obsigate.tools.spreadsheets")

# xlsx_to_markdown ceiling: a workbook is a data dump, not prose. The table is
# for the LLM context, so both axes are bounded (same spirit as A5's index cap).
MAX_MD_ROWS = 100
MAX_MD_COLS = 20
MAX_MD_CHARS = 20_000

# #156-A14 — the newer read tools scan more than the markdown table (a search
# or an aggregate must not stop at row 100) but stay bounded all the same:
# a runaway scan would load a whole ledger into the model's context.
MAX_SCAN_ROWS = 5_000
MAX_SCAN_COLS = 100
MAX_SEARCH_RESULTS = 100
MAX_RANGE_CELLS = 10_000
MAX_RANGE_VALUES = 200

# The formats the spreadsheet editor (and now the assistant) can handle.
SPREADSHEET_EXTENSIONS = (".xlsx", ".xlsm", ".csv")

_NUMBER_RE = re.compile(r"^-?\d+(?:[.,]\d+)?$")


def _spreadsheet_path(vault: str, path: str) -> Path:
    """Resolve and validate a vault-relative ``.xlsx``/``.xlsm``/``.csv`` path."""
    path = (path or "").strip()
    if not path.lower().endswith(SPREADSHEET_EXTENSIONS):
        raise ToolError(
            "Extension attendue : .xlsx, .xlsm ou .csv", code="invalid_arguments"
        )
    try:
        root = get_vault_root(vault)
    except ServiceError as e:
        raise ToolError(e.message, code=e.code, details=e.details) from e
    return resolve_safe_path(root, path)


def _is_csv(file_path: Path) -> bool:
    return file_path.suffix.lower() == ".csv"


def _map_service_error(e: ServiceError) -> ToolError:
    return ToolError(e.message, code=e.code, details=e.details)


def _sheet_titles(file_path: Path) -> list[str]:
    """Sheet names of a workbook; a CSV has a single, unnamed “sheet”."""
    if _is_csv(file_path):
        return [""]
    from openpyxl import load_workbook

    wb = load_workbook(str(file_path), read_only=True, data_only=True)
    try:
        return list(wb.sheetnames)
    finally:
        wb.close()


def _read_grid(
    file_path: Path, sheet: str, max_rows: int, max_cols: int
) -> tuple[str, list[list[str]], bool]:
    """Read one sheet (or a CSV) as bounded, formatted, trimmed rows.

    Returns ``(title, rows, truncated)``; ``truncated`` is True when real data
    sits just beyond the row cap (probed one row further) so the caller can say
    so instead of silently dropping it.
    """
    from openpyxl import load_workbook

    from backend.xlsx_reader import _fmt

    if _is_csv(file_path):
        import csv as csv_mod
        import io as io_mod

        from backend.xlsx_reader import sniff_csv_delimiter

        stem = file_path.stem
        if sheet and sheet != stem:
            raise ToolError(f"Feuille introuvable: {sheet}", code="not_found")
        raw = file_path.read_text(encoding="utf-8-sig", errors="replace")
        reader = csv_mod.reader(io_mod.StringIO(raw), delimiter=sniff_csv_delimiter(raw))
        grid: list[list[str]] = []
        truncated = False
        for i, row in enumerate(reader):
            if i >= max_rows:
                truncated = any(str(c).strip() for c in row)
                break
            grid.append([str(c) for c in row][:max_cols])
        while grid and not any(c.strip() for c in grid[-1]):
            grid.pop()
        return stem, grid, truncated

    try:
        wb = load_workbook(str(file_path), read_only=True, data_only=True)
    except ServiceError as e:
        raise _map_service_error(e) from e
    except Exception as e:
        raise ToolError(f"Classeur illisible: {e}", code="invalid") from e
    try:
        if sheet:
            if sheet not in wb.sheetnames:
                raise ToolError(f"Feuille introuvable: {sheet}", code="not_found")
            ws = wb[sheet]
        else:
            ws = wb.active
        title = ws.title
        grid = []
        for row in ws.iter_rows(
            min_row=1, max_row=max_rows, max_col=max_cols, values_only=True
        ):
            grid.append([_fmt(v) for v in row])
        probe = list(
            ws.iter_rows(
                min_row=max_rows + 1,
                max_row=max_rows + 1,
                max_col=max_cols,
                values_only=True,
            )
        )
        truncated = any(any(str(v or "").strip() for v in r) for r in probe)
    finally:
        wb.close()
    while grid and not any(c.strip() for c in grid[-1]):
        grid.pop()
    return title, grid, truncated


def _to_number(text: str) -> float | None:
    """Coerce a displayed cell to a float, or ``None`` when it is not one."""
    t = text.strip().replace("\u00a0", "").replace(" ", "")
    if not _NUMBER_RE.match(t):
        return None
    try:
        return float(t.replace(",", "."))
    except ValueError:  # pragma: no cover - regex already guarantees the shape
        return None


@tool(
    name="list_xlsx_sheets",
    description=(
        "List the sheets of a spreadsheet (.xlsx, .xlsm or .csv) with their "
        "dimensions (rows x columns) and whether the display caps truncate "
        "them. Use before editing to pick the right sheet name."
    ),
    input_model=ListXlsxSheetsInput,
    risk=ToolRisk.READ,
    requires_vault=True,
)
def list_xlsx_sheets(ctx: ToolContext, params: ListXlsxSheetsInput) -> dict[str, Any]:
    """Return sheet names and extents of the workbook (or of the CSV)."""
    from backend.xlsx_reader import MAX_COLS, MAX_ROWS

    file_path = _spreadsheet_path(params.vault, params.path)
    if not file_path.exists() or not file_path.is_file():
        raise ToolError(f"Fichier introuvable: {params.path}", code="not_found")

    extents: list[tuple[str, int, int]] = []
    if _is_csv(file_path):
        _, rows, _ = _read_grid(file_path, "", MAX_SCAN_ROWS, MAX_SCAN_COLS)
        extents.append(
            (
                file_path.stem,
                len(rows),
                max((len(r) for r in rows), default=0),
            )
        )
    else:
        # Declared dimensions are enough here (and far cheaper than scanning
        # every row): the caller just needs a size to decide what to read.
        from openpyxl import load_workbook

        from backend.xlsx_reader import _sheet_extent

        try:
            wb = load_workbook(str(file_path), read_only=True, data_only=True)
        except ServiceError as e:
            raise _map_service_error(e) from e
        except Exception as e:
            raise ToolError(f"Classeur illisible: {e}", code="invalid") from e
        try:
            extents = [
                (ws.title, *_sheet_extent(ws)) for ws in wb.worksheets
            ]
        finally:
            wb.close()

    sheets = [
        {
            "name": name,
            "total_rows": total_rows,
            "total_cols": total_cols,
            "truncated": total_rows > MAX_ROWS or total_cols > MAX_COLS,
        }
        for name, total_rows, total_cols in extents
    ]
    return {"vault": params.vault, "path": params.path, "sheets": sheets}


@tool(
    name="xlsx_to_markdown",
    description=(
        "Read a sheet of a spreadsheet (.xlsx, .xlsm or .csv) as a bounded "
        "markdown table (up to 100 rows x 20 columns). Use to inspect "
        "spreadsheet data before answering or editing."
    ),
    input_model=XlsxToMarkdownInput,
    risk=ToolRisk.READ,
    requires_vault=True,
)
def xlsx_to_markdown(ctx: ToolContext, params: XlsxToMarkdownInput) -> dict[str, Any]:
    """Render one sheet as a markdown table for the LLM context."""
    file_path = _spreadsheet_path(params.vault, params.path)
    title, rows, truncated = _read_grid(file_path, params.sheet, MAX_MD_ROWS, MAX_MD_COLS)

    lines: list[str] = []
    if rows:
        header = rows[0]
        lines.append("| " + " | ".join(header) + " |")
        lines.append("|" + "|".join("---" for _ in header) + "|")
        for row in rows[1:]:
            lines.append("| " + " | ".join(row) + " |")
    table = "\n".join(lines)[:MAX_MD_CHARS]

    return {
        "vault": params.vault,
        "path": params.path,
        "sheet": title,
        "rows": len(rows),
        "cols": max((len(r) for r in rows), default=0),
        "truncated": truncated,
        "markdown": table,
    }


@tool(
    name="search_workbook",
    description=(
        "Search a text across every sheet of a spreadsheet (.xlsx, .xlsm or "
        ".csv) and return the matching cells with their sheet and A1 "
        "reference (max 100 matches). Use it to find where a value lives "
        "without dumping whole sheets into the context."
    ),
    input_model=SearchWorkbookInput,
    risk=ToolRisk.READ,
    requires_vault=True,
)
def search_workbook(ctx: ToolContext, params: SearchWorkbookInput) -> dict[str, Any]:
    """Find a needle across all sheets, bounded and counted per sheet."""
    from openpyxl.utils import get_column_letter

    file_path = _spreadsheet_path(params.vault, params.path)
    needle = (params.query or "").strip()
    if not needle:
        raise ToolError("Requête vide", code="invalid_arguments")

    titles = _sheet_titles(file_path)
    if params.sheet:
        if params.sheet not in titles:
            raise ToolError(f"Feuille introuvable: {params.sheet}", code="not_found")
        titles = [params.sheet]

    hay = needle if params.case_sensitive else needle.lower()
    matches: list[dict[str, Any]] = []
    by_sheet: dict[str, int] = {}
    total = 0
    for title in titles:
        sheet_title, rows, _ = _read_grid(
            file_path, title, MAX_SCAN_ROWS, MAX_SCAN_COLS
        )
        label = sheet_title or file_path.stem
        for r_i, row in enumerate(rows, start=1):
            for c_i, value in enumerate(row, start=1):
                if not value:
                    continue
                haystack = value if params.case_sensitive else value.lower()
                if hay not in haystack:
                    continue
                total += 1
                by_sheet[label] = by_sheet.get(label, 0) + 1
                if len(matches) < MAX_SEARCH_RESULTS:
                    matches.append(
                        {
                            "sheet": label,
                            "cell": f"{get_column_letter(c_i)}{r_i}",
                            "value": value,
                        }
                    )
    return {
        "vault": params.vault,
        "path": params.path,
        "query": needle,
        "total": total,
        "truncated": total > MAX_SEARCH_RESULTS,
        "by_sheet": by_sheet,
        "matches": matches,
    }


@tool(
    name="analyze_range",
    description=(
        "Aggregate an A1 range of a sheet (.xlsx, .xlsm or .csv): count, sum, "
        "mean, min and max of the numeric cells, plus a bounded sample of the "
        "values. Use it to answer a question about a column without reading "
        "the whole sheet."
    ),
    input_model=AnalyzeRangeInput,
    risk=ToolRisk.READ,
    requires_vault=True,
)
def analyze_range(ctx: ToolContext, params: AnalyzeRangeInput) -> dict[str, Any]:
    """Numeric aggregates + value sample over an A1 range of the sheet."""
    from openpyxl.utils.cell import range_boundaries

    file_path = _spreadsheet_path(params.vault, params.path)
    title, rows, _ = _read_grid(file_path, params.sheet, MAX_SCAN_ROWS, MAX_SCAN_COLS)

    label = (params.range or "").strip()
    if label:
        try:
            min_col, min_row, max_col, max_row = range_boundaries(label.upper())
        except Exception as e:
            raise ToolError(f"Plage invalide: {label}", code="invalid_arguments") from e
        if (max_row - min_row + 1) * (max_col - min_col + 1) > MAX_RANGE_CELLS:
            raise ToolError(
                f"Plage trop grande (max {MAX_RANGE_CELLS} cellules)",
                code="invalid_arguments",
            )
        selected = [row[min_col - 1 : max_col] for row in rows[min_row - 1 : max_row]]
    else:
        selected = rows

    values = [v for row in selected for v in row if isinstance(v, str) and v.strip()]
    numbers = [n for n in (_to_number(v) for v in values) if n is not None]

    stats: dict[str, Any] = {"count": len(numbers)}
    if numbers:
        stats.update(
            {
                "sum": round(sum(numbers), 6),
                "mean": round(sum(numbers) / len(numbers), 6),
                "min": min(numbers),
                "max": max(numbers),
            }
        )
    return {
        "vault": params.vault,
        "path": params.path,
        "sheet": title,
        "range": params.range or "",
        "rows": len(selected),
        "cols": max((len(r) for r in selected), default=0),
        "cells": len(values),
        "numeric": stats,
        "values": values[:MAX_RANGE_VALUES],
        "truncated": len(values) > MAX_RANGE_VALUES,
    }


@tool(
    name="update_xlsx_cells",
    description=(
        "Edit cells of an existing spreadsheet (.xlsx, .xlsm or .csv). "
        "``cells`` maps A1 references to new values (max 500). A value "
        "starting with '=' or '@' is stored as TEXT unless allow_formula is "
        "set (DDE guard). Editing a workbook carrying features openpyxl "
        "cannot rewrite requires force=true (cached formula results, "
        "slicers…). A .csv has no sheet: any ``sheet`` value is ignored."
    ),
    input_model=UpdateXlsxCellsInput,
    risk=ToolRisk.WRITE,
    requires_vault=True,
)
def update_xlsx_cells(ctx: ToolContext, params: UpdateXlsxCellsInput) -> dict[str, Any]:
    """Wrap the guarded cell-edit service (``.xlsx``/``.xlsm`` or ``.csv``)."""
    from backend.services.mutations import edit_xlsx_cells, save_csv_cells

    file_path = _spreadsheet_path(params.vault, params.path)
    if not params.cells:
        raise ToolError("Aucune cellule fournie", code="invalid_arguments")
    if not _is_csv(file_path) and not params.sheet:
        raise ToolError("Feuille requise pour un classeur", code="invalid_arguments")
    try:
        if _is_csv(file_path):
            result = save_csv_cells(params.vault, params.path, dict(params.cells))
        else:
            result = edit_xlsx_cells(
                params.vault,
                params.path,
                params.sheet,
                dict(params.cells),
                allow_formula=params.allow_formula,
                force=params.force,
            )
    except ServiceError as e:
        raise _map_service_error(e) from e
    return {
        "status": "ok",
        "vault": result["vault"],
        "path": result["path"],
        "sheet": params.sheet,
        "cells": len(params.cells),
    }


@tool(
    name="append_xlsx_rows",
    description=(
        "Append rows at the end of a sheet of an existing spreadsheet "
        "(.xlsx or .xlsm). Values are typed like in the viewer (numbers, "
        "TRUE/FALSE, FR dates JJ/MM/AAAA). The workbook is rewritten "
        "atomically with a backup. Not available for .csv."
    ),
    input_model=AppendXlsxRowsInput,
    risk=ToolRisk.WRITE,
    requires_vault=True,
)
def append_xlsx_rows(ctx: ToolContext, params: AppendXlsxRowsInput) -> dict[str, Any]:
    """Append whole rows below the last used row of the sheet."""
    from openpyxl import load_workbook
    from openpyxl.utils import get_column_letter

    from backend.services.mutations import _coerce_xlsx_value, edit_xlsx_cells

    if not params.rows:
        raise ToolError("Aucune ligne fournie", code="invalid_arguments")
    if len(params.rows) > 500:
        raise ToolError("Trop de lignes (max 500)", code="invalid_arguments")

    file_path = _spreadsheet_path(params.vault, params.path)
    if _is_csv(file_path):
        raise ToolError(
            "Un .csv n'a pas de notion de fin de feuille : utilisez "
            "update_xlsx_cells avec des références A1",
            code="invalid_arguments",
        )
    try:
        wb = load_workbook(str(file_path), read_only=True, data_only=True)
        try:
            if params.sheet not in wb.sheetnames:
                raise ToolError(
                    f"Feuille introuvable: {params.sheet}", code="not_found"
                )
            ws = wb[params.sheet]
            first_free = (ws.max_row or 0) + 1
        finally:
            wb.close()
    except ServiceError as e:
        raise _map_service_error(e) from e
    except ToolError:
        raise
    except Exception as e:
        raise ToolError(f"Classeur illisible: {e}", code="invalid") from e

    cells: dict[str, Any] = {}
    for i, row in enumerate(params.rows):
        for j, value in enumerate(row):
            if value is None or (isinstance(value, str) and not value.strip()):
                continue
            ref = f"{get_column_letter(j + 1)}{first_free + i}"
            cells[ref] = _coerce_xlsx_value(value)
    if not cells:
        raise ToolError("Aucune valeur fournie", code="invalid_arguments")

    try:
        result = edit_xlsx_cells(
            params.vault,
            params.path,
            params.sheet,
            cells,
            allow_formula=params.allow_formula,
            force=params.force,
        )
    except ServiceError as e:
        raise _map_service_error(e) from e
    return {
        "status": "ok",
        "vault": result["vault"],
        "path": result["path"],
        "sheet": params.sheet,
        "rows": len(params.rows),
        "first_row": first_free,
    }


@tool(
    name="edit_xlsx_structure",
    description=(
        "Change the structure of an existing .xlsx/.xlsm workbook: add, "
        "rename, duplicate or delete a sheet, or insert/delete rows and "
        "columns. ``actions`` is an ordered list of "
        '{"op": "sheet_add"|"sheet_rename"|"sheet_duplicate"|"sheet_delete"|'
        '"row_insert"|"row_delete"|"col_insert"|"col_delete", …} '
        "(1 to 50). Deletions drop data and cannot be undone from the "
        "assistant — confirm with the user first."
    ),
    input_model=EditXlsxStructureInput,
    risk=ToolRisk.WRITE,
    requires_vault=True,
)
def edit_xlsx_structure(
    ctx: ToolContext, params: EditXlsxStructureInput
) -> dict[str, Any]:
    """Apply a batch of structural changes through the guarded service."""
    from backend.services.mutations import mutate_xlsx_structure

    if not params.actions:
        raise ToolError("Aucune action fournie", code="invalid_arguments")
    if len(params.actions) > 50:
        raise ToolError("Trop d'actions (max 50)", code="invalid_arguments")
    if str(params.path or "").lower().endswith(".csv"):
        raise ToolError(
            "Un .csv n'a pas de structure modifiable", code="invalid_arguments"
        )
    try:
        result = mutate_xlsx_structure(
            params.vault, params.path, [dict(a) for a in params.actions], force=params.force
        )
    except ServiceError as e:
        raise _map_service_error(e) from e
    return {
        "status": "ok",
        "vault": result["vault"],
        "path": result["path"],
        "actions": len(params.actions),
    }