216 lines
7.7 KiB
Python
216 lines
7.7 KiB
Python
"""Duplicate detection & merge services (#166).
|
|
|
|
Single source of truth consumed by the REST routes
|
|
(``/api/duplicates``) and the AI tool layer (``find_duplicates``,
|
|
``merge_duplicate_notes``).
|
|
|
|
Method is deterministic stdlib-only: frontmatter stripped, token-set
|
|
Jaccard blended with a title similarity. No embedding dependency —
|
|
the semantic index (#70) stays an optional refinement, not a requirement.
|
|
|
|
Fusion never runs without an explicit confirmation: the tool layer
|
|
registers the merge as ``DANGEROUS`` (two-step propose/apply) and this
|
|
service takes an automatic backup before any destructive write.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
from difflib import SequenceMatcher
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from backend.services.backups import create_backup
|
|
from backend.services.errors import ServiceError
|
|
from backend.services.paths import resolve_safe_path
|
|
from backend.services.vaults import get_vault_root
|
|
|
|
logger = logging.getLogger("obsigate.services.duplicates")
|
|
|
|
MAX_FILES_SCANNED = 500
|
|
MAX_FILE_BYTES = 200_000
|
|
MAX_CONTENT_CHARS = 50_000
|
|
|
|
_WORD_RE = re.compile(r"[\w]+", re.UNICODE)
|
|
_FRONTMATTER_RE = re.compile(r"\A---\s*\n.*?\n---\s*\n", re.DOTALL)
|
|
|
|
|
|
def _strip_frontmatter(text: str) -> str:
|
|
"""Remove a leading YAML frontmatter block, if present."""
|
|
return _FRONTMATTER_RE.sub("", text, count=1)
|
|
|
|
|
|
def _tokens(text: str) -> set[str]:
|
|
"""Lowercase word tokens (keeps accents), stop-words free but tiny tokens dropped."""
|
|
return {t for t in _WORD_RE.findall(text.lower()) if len(t) > 2}
|
|
|
|
|
|
def similarity_score(a: str, b: str) -> float:
|
|
"""Blend Jaccard (0.7) + title/first-line similarity (0.3) in [0, 1].
|
|
|
|
Pure function — unit-tested directly.
|
|
"""
|
|
ta, tb = _tokens(_strip_frontmatter(a)), _tokens(_strip_frontmatter(b))
|
|
if not ta or not tb:
|
|
return 0.0
|
|
jaccard = len(ta & tb) / len(ta | tb)
|
|
head_a = (a.strip().splitlines() or [""])[:1][0][:200].lower()
|
|
head_b = (b.strip().splitlines() or [""])[:1][0][:200].lower()
|
|
title_sim = SequenceMatcher(None, head_a, head_b).ratio() if head_a and head_b else 0.0
|
|
return round(0.7 * jaccard + 0.3 * title_sim, 4)
|
|
|
|
|
|
def _iter_markdown_files(root: Path, subdir: str = "") -> list[Path]:
|
|
base = resolve_safe_path(root, subdir) if subdir else root.resolve()
|
|
if not base.exists() or not base.is_dir():
|
|
raise ServiceError(
|
|
f"Directory not found: {subdir or '.'}",
|
|
code="not_found",
|
|
status=404,
|
|
details={"path": subdir},
|
|
)
|
|
files = sorted(
|
|
(p for p in base.rglob("*.md") if p.is_file() and not p.is_symlink()),
|
|
key=lambda p: str(p),
|
|
)
|
|
return files[:MAX_FILES_SCANNED]
|
|
|
|
|
|
def _read_capped(path: Path) -> str:
|
|
try:
|
|
if path.stat().st_size > MAX_FILE_BYTES:
|
|
return ""
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return ""
|
|
return text[:MAX_CONTENT_CHARS]
|
|
|
|
|
|
def find_duplicate_pairs(
|
|
vault: str,
|
|
threshold: float = 0.75,
|
|
limit: int = 50,
|
|
subdir: str = "",
|
|
) -> dict[str, Any]:
|
|
"""Return candidate duplicate pairs ordered by descending score.
|
|
|
|
Args:
|
|
vault: Vault name.
|
|
threshold: Minimum blended score in [0.3, 1.0].
|
|
limit: Max pairs returned (1-200).
|
|
subdir: Optional vault-relative directory scope.
|
|
"""
|
|
if not 0.3 <= threshold <= 1.0:
|
|
raise ServiceError(
|
|
"threshold must be between 0.3 and 1.0",
|
|
code="invalid_arguments",
|
|
status=400,
|
|
)
|
|
limit = max(1, min(limit, 200))
|
|
root = get_vault_root(vault)
|
|
files = _iter_markdown_files(root, subdir)
|
|
contents: dict[str, str] = {}
|
|
token_sets: dict[str, set[str]] = {}
|
|
for path in files:
|
|
rel = str(path.relative_to(root)).replace("\\", "/")
|
|
text = _read_capped(path)
|
|
if not text.strip():
|
|
continue
|
|
contents[rel] = text
|
|
token_sets[rel] = _tokens(_strip_frontmatter(text))
|
|
|
|
rels = sorted(contents)
|
|
pairs: list[dict[str, Any]] = []
|
|
for i in range(len(rels)):
|
|
for j in range(i + 1, len(rels)):
|
|
a, b = rels[i], rels[j]
|
|
ta, tb = token_sets[a], token_sets[b]
|
|
if not ta or not tb:
|
|
continue
|
|
# Cheap pre-filter: Jaccard lower bound before the full score.
|
|
inter = len(ta & tb)
|
|
union = len(ta | tb)
|
|
if union == 0 or inter / union < threshold * 0.6:
|
|
continue
|
|
score = similarity_score(contents[a], contents[b])
|
|
if score >= threshold:
|
|
pairs.append({"file_a": a, "file_b": b, "score": score})
|
|
pairs.sort(key=lambda p: p["score"], reverse=True)
|
|
return {
|
|
"vault": vault,
|
|
"threshold": threshold,
|
|
"files_scanned": len(contents),
|
|
"truncated": len(files) >= MAX_FILES_SCANNED,
|
|
"pairs": pairs[:limit],
|
|
}
|
|
|
|
|
|
def merge_duplicates(
|
|
vault: str,
|
|
source_path: str,
|
|
target_path: str,
|
|
strategy: str = "append",
|
|
) -> dict[str, Any]:
|
|
"""Merge *source_path* into *target_path*, then delete the source.
|
|
|
|
Strategies:
|
|
``append`` — source content appended after target (separator + origin
|
|
marker), source deleted.
|
|
``prefer_target`` — source deleted, target untouched (dedupe only).
|
|
``prefer_source`` — target overwritten with source content, source deleted.
|
|
|
|
A backup of both files is taken first; the source deletion also goes
|
|
through the backup-aware mutation service.
|
|
"""
|
|
from backend.services import mutations as _mutations
|
|
|
|
if strategy not in ("append", "prefer_target", "prefer_source"):
|
|
raise ServiceError(
|
|
f"Unknown strategy: {strategy}",
|
|
code="invalid_arguments",
|
|
status=400,
|
|
)
|
|
if source_path == target_path:
|
|
raise ServiceError(
|
|
"source_path and target_path must differ",
|
|
code="invalid_arguments",
|
|
status=400,
|
|
)
|
|
root = get_vault_root(vault)
|
|
src = resolve_safe_path(root, source_path)
|
|
dst = resolve_safe_path(root, target_path)
|
|
if not src.is_file() or src.suffix.lower() != ".md":
|
|
raise ServiceError(
|
|
f"Source not found: {source_path}",
|
|
code="not_found",
|
|
status=404,
|
|
details={"path": source_path},
|
|
)
|
|
if not dst.is_file() or dst.suffix.lower() != ".md":
|
|
raise ServiceError(
|
|
f"Target not found: {target_path}",
|
|
code="not_found",
|
|
status=404,
|
|
details={"path": target_path},
|
|
)
|
|
# Backup préalable (jamais de fusion sans filet — critère #166).
|
|
create_backup(src, vault, source_path)
|
|
create_backup(dst, vault, target_path)
|
|
|
|
if strategy == "prefer_target":
|
|
result = _mutations.delete_file(vault, source_path)
|
|
return {"strategy": strategy, "target": target_path, "deleted": source_path, "delete": result}
|
|
if strategy == "prefer_source":
|
|
content = src.read_text(encoding="utf-8", errors="replace")
|
|
result = _mutations.edit_file(vault, target_path, content)
|
|
deleted = _mutations.delete_file(vault, source_path)
|
|
return {"strategy": strategy, "target": target_path, "edit": result, "deleted": source_path, "delete": deleted}
|
|
# append
|
|
target_text = dst.read_text(encoding="utf-8", errors="replace")
|
|
source_text = src.read_text(encoding="utf-8", errors="replace")
|
|
merged = target_text.rstrip() + f"\n\n---\n\n_Fusionné depuis `{source_path}` (#166)_\n\n" + source_text.lstrip()
|
|
result = _mutations.edit_file(vault, target_path, merged)
|
|
deleted = _mutations.delete_file(vault, source_path)
|
|
return {"strategy": strategy, "target": target_path, "edit": result, "deleted": source_path, "delete": deleted}
|