feat: agent IA phase 4 — doublons #166, notifications externes #168 (Discord/Telegram/SMTP/webhook), taches planifiees #170
This commit is contained in:
@@ -368,6 +368,24 @@ async def lifespan(app: FastAPI):
|
||||
|
||||
asyncio.create_task(_background_startup())
|
||||
|
||||
async def _scheduler_loop():
|
||||
"""Background tick for scheduled tasks (#170) — every 60 s, best effort."""
|
||||
from backend.scheduler import tick
|
||||
|
||||
await asyncio.sleep(60)
|
||||
while True:
|
||||
try:
|
||||
outcomes = await asyncio.to_thread(tick)
|
||||
if outcomes:
|
||||
logger.info(f"Scheduler tick: {len(outcomes)} task(s) executed")
|
||||
except Exception as e:
|
||||
logger.warning(f"Scheduler tick failed: {e}")
|
||||
await asyncio.sleep(60)
|
||||
|
||||
if os.environ.get("OBSIGATE_SCHEDULER", "1") != "0":
|
||||
asyncio.create_task(_scheduler_loop())
|
||||
logger.info("Scheduler loop started (#170, 60 s tick).")
|
||||
|
||||
logger.info("ObsiGate ready (listening for requests while indexing).")
|
||||
yield
|
||||
|
||||
@@ -490,12 +508,15 @@ from backend.routers.backups import router as backups_router
|
||||
from backend.routers.config import _load_config
|
||||
from backend.routers.config import router as config_router
|
||||
from backend.routers.conflicts import router as conflicts_router
|
||||
from backend.routers.duplicates import router as duplicates_router
|
||||
from backend.routers.files_media import router as files_media_router
|
||||
from backend.routers.files_read import router as files_read_router
|
||||
from backend.routers.files_write import router as files_write_router
|
||||
from backend.routers.health import router as health_router
|
||||
from backend.routers.history import router as history_router
|
||||
from backend.routers.notify import router as notify_router
|
||||
from backend.routers.realtime import router as realtime_router
|
||||
from backend.routers.scheduler import router as scheduler_router
|
||||
from backend.routers.search import router as search_router
|
||||
from backend.routers.sharing import router as sharing_router
|
||||
from backend.routers.vaults import router as vaults_router
|
||||
@@ -519,6 +540,9 @@ app.include_router(files_write_router) # ROADMAP #85 T6b — Files write
|
||||
app.include_router(webhooks_router) # ROADMAP #85 T2 — Webhooks
|
||||
app.include_router(sharing_router) # ROADMAP #85 T3 — Sharing
|
||||
app.include_router(vaults_router) # ROADMAP #85 T8 — Vaults
|
||||
app.include_router(duplicates_router) # ROADMAP #166 — Doublons
|
||||
app.include_router(notify_router) # ROADMAP #168 — Notifications externes
|
||||
app.include_router(scheduler_router) # ROADMAP #170 — Tâches planifiées
|
||||
|
||||
# Admin Dashboard endpoints (system stats, audit logs, backups, stream)
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
"""External notifications — Discord, Telegram, SMTP, generic webhook (#168).
|
||||
|
||||
Configuration is persisted in ``data/notify_channels.json``; secrets live in
|
||||
``data/notify_secrets.json`` (0600) or in ``OBSIGATE_NOTIFY_SECRET_<ID>``
|
||||
environment variables — never in the public config file (same pattern as
|
||||
``backend/webhooks.py``, BUG-026).
|
||||
|
||||
Supported channel types:
|
||||
|
||||
* ``discord`` — Discord webhook URL (``POST {"content": ...}``).
|
||||
* ``telegram`` — Bot API (``POST https://api.telegram.org/bot<token>/sendMessage``).
|
||||
* ``smtp`` — Email via stdlib ``smtplib`` (STARTTLS, auth login).
|
||||
* ``webhook`` — generic JSON ``POST`` (SSRF-safe, same policy as #9).
|
||||
|
||||
Each channel declares ``triggers`` chosen among :data:`VALID_TRIGGERS`.
|
||||
The scheduler (#170) broadcasts on ``schedule_failure``; file-event fan-out
|
||||
stays on the historical ``backend/webhooks.py`` path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import smtplib
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from email.message import EmailMessage
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("obsigate.notify")
|
||||
|
||||
DATA_DIR = Path(os.environ.get("OBSIGATE_DATA_DIR", "data"))
|
||||
CHANNELS_FILE = DATA_DIR / "notify_channels.json"
|
||||
SECRETS_FILE = DATA_DIR / "notify_secrets.json"
|
||||
|
||||
CHANNEL_TYPES = ("discord", "telegram", "smtp", "webhook")
|
||||
VALID_TRIGGERS = ("manual", "schedule_failure", "schedule_success", "duplicate_found")
|
||||
|
||||
_lock = threading.RLock()
|
||||
|
||||
|
||||
# ── Store helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _read_channels() -> list[dict[str, Any]]:
|
||||
if not CHANNELS_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(CHANNELS_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return []
|
||||
|
||||
|
||||
def _write_channels(channels: list[dict[str, Any]]) -> None:
|
||||
CHANNELS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CHANNELS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(channels, indent=2, default=str), encoding="utf-8")
|
||||
tmp.replace(CHANNELS_FILE)
|
||||
|
||||
|
||||
def _read_secrets() -> dict[str, str]:
|
||||
if not SECRETS_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(SECRETS_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return {}
|
||||
|
||||
|
||||
def _write_secrets(secrets: dict[str, str]) -> None:
|
||||
SECRETS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = SECRETS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(secrets, indent=2), encoding="utf-8")
|
||||
tmp.replace(SECRETS_FILE)
|
||||
try:
|
||||
SECRETS_FILE.chmod(0o600)
|
||||
except OSError:
|
||||
pass # Windows: pas de permissions Unix
|
||||
|
||||
|
||||
def _secret_key(channel_id: str) -> str:
|
||||
return "OBSIGATE_NOTIFY_SECRET_" + channel_id.replace("-", "_").upper()
|
||||
|
||||
|
||||
def _get_secret(channel_id: str) -> str | None:
|
||||
"""Resolve a channel secret: env > dedicated store > legacy inline config."""
|
||||
env_val = os.environ.get(_secret_key(channel_id))
|
||||
if env_val:
|
||||
return env_val
|
||||
stored = _read_secrets().get(channel_id)
|
||||
if stored:
|
||||
return stored
|
||||
for ch in _read_channels():
|
||||
if ch.get("id") == channel_id:
|
||||
cfg = ch.get("config", {})
|
||||
for key in ("webhook_url", "bot_token", "password"):
|
||||
if cfg.get(key):
|
||||
return str(cfg[key])
|
||||
return None
|
||||
|
||||
|
||||
def _public_view(channel: dict[str, Any]) -> dict[str, Any]:
|
||||
clean = {k: v for k, v in channel.items() if k != "config"}
|
||||
cfg = dict(channel.get("config", {}))
|
||||
for secret_field in ("webhook_url", "bot_token", "password"):
|
||||
if cfg.get(secret_field):
|
||||
cfg[secret_field] = "***"
|
||||
clean["config"] = cfg
|
||||
clean["has_secret"] = bool(_get_secret(channel["id"]))
|
||||
return clean
|
||||
|
||||
|
||||
# ── CRUD ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _validate_config(channel_type: str, config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Validate (sans secret) and normalize a channel config. Raises ValueError."""
|
||||
config = dict(config or {})
|
||||
if channel_type == "discord":
|
||||
url = str(config.get("webhook_url") or config.get("url") or "").strip()
|
||||
if not url.startswith(("https://discord.com/api/webhooks/", "https://discordapp.com/api/webhooks/")):
|
||||
# Laisse passer les URLs de test locales quand le mode privé est ouvert.
|
||||
from backend.webhooks import validate_webhook_url
|
||||
|
||||
validate_webhook_url(url)
|
||||
if "discord" not in url and not os.environ.get("OBSIGATE_WEBHOOK_ALLOW_PRIVATE"):
|
||||
raise ValueError("URL Discord invalide (webhook discord.com attendu)")
|
||||
config["webhook_url"] = url
|
||||
elif channel_type == "telegram":
|
||||
if not str(config.get("chat_id") or "").strip():
|
||||
raise ValueError("chat_id Telegram requis")
|
||||
config["chat_id"] = str(config["chat_id"]).strip()
|
||||
if config.get("bot_token"):
|
||||
config["bot_token"] = str(config["bot_token"]).strip()
|
||||
elif channel_type == "smtp":
|
||||
for field in ("host", "from_addr", "to_addr"):
|
||||
if not str(config.get(field) or "").strip():
|
||||
raise ValueError(f"Champ SMTP requis : {field}")
|
||||
config["port"] = int(config.get("port") or 587)
|
||||
config["use_tls"] = bool(config.get("use_tls", True))
|
||||
config["username"] = str(config.get("username") or "").strip()
|
||||
elif channel_type == "webhook":
|
||||
from backend.webhooks import validate_webhook_url
|
||||
|
||||
url = str(config.get("url") or "").strip()
|
||||
validate_webhook_url(url)
|
||||
config["url"] = url
|
||||
else:
|
||||
raise ValueError(f"Type de canal inconnu : {channel_type}")
|
||||
triggers = [t for t in (config.get("triggers") or ["manual"]) if t in VALID_TRIGGERS]
|
||||
config["triggers"] = triggers or ["manual"]
|
||||
return config
|
||||
|
||||
|
||||
def list_channels() -> list[dict[str, Any]]:
|
||||
"""Return public views of all notification channels."""
|
||||
return [_public_view(ch) for ch in _read_channels()]
|
||||
|
||||
|
||||
def create_channel(name: str, channel_type: str, config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create a notification channel. Secrets are split into the secret store."""
|
||||
if channel_type not in CHANNEL_TYPES:
|
||||
raise ValueError(f"Type de canal inconnu : {channel_type}")
|
||||
with _lock:
|
||||
channels = _read_channels()
|
||||
channel_id = str(uuid.uuid4())
|
||||
normalized = _validate_config(channel_type, config)
|
||||
secrets = _read_secrets()
|
||||
for field in ("webhook_url", "bot_token", "password"):
|
||||
if normalized.get(field) and len(str(normalized[field])) > 8:
|
||||
secrets[channel_id] = str(normalized[field])
|
||||
normalized[field] = "***" # placeholder : le secret vit dans le store dédié
|
||||
_write_secrets(secrets)
|
||||
channel = {
|
||||
"id": channel_id,
|
||||
"name": (name or channel_type).strip() or channel_type,
|
||||
"type": channel_type,
|
||||
"enabled": True,
|
||||
"config": normalized,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"last_sent_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
channels.append(channel)
|
||||
_write_channels(channels)
|
||||
logger.info(f"Created notify channel '{name}' ({channel_type})")
|
||||
return _public_view(channel)
|
||||
|
||||
|
||||
def update_channel(channel_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update a channel (name/enabled/config). Returns None when unknown."""
|
||||
with _lock:
|
||||
channels = _read_channels()
|
||||
for channel in channels:
|
||||
if channel.get("id") != channel_id:
|
||||
continue
|
||||
if updates.get("name"):
|
||||
channel["name"] = str(updates["name"])
|
||||
if "enabled" in updates:
|
||||
channel["enabled"] = bool(updates["enabled"])
|
||||
if "config" in updates and isinstance(updates["config"], dict):
|
||||
merged = {**channel.get("config", {}), **updates["config"]}
|
||||
normalized = _validate_config(channel["type"], merged)
|
||||
secrets = _read_secrets()
|
||||
for field in ("webhook_url", "bot_token", "password"):
|
||||
if updates["config"].get(field):
|
||||
secrets[channel_id] = str(updates["config"][field])
|
||||
normalized[field] = "***"
|
||||
_write_secrets(secrets)
|
||||
channel["config"] = normalized
|
||||
_write_channels(channels)
|
||||
return _public_view(channel)
|
||||
return None
|
||||
|
||||
|
||||
def delete_channel(channel_id: str) -> bool:
|
||||
"""Delete a channel and its secret. Returns False when unknown."""
|
||||
with _lock:
|
||||
channels = _read_channels()
|
||||
remaining = [c for c in channels if c.get("id") != channel_id]
|
||||
if len(remaining) == len(channels):
|
||||
return False
|
||||
_write_channels(remaining)
|
||||
secrets = _read_secrets()
|
||||
if secrets.pop(channel_id, None) is not None:
|
||||
_write_secrets(secrets)
|
||||
return True
|
||||
|
||||
|
||||
# ── Dispatch ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _send_discord(webhook_url: str, title: str, message: str) -> None:
|
||||
import httpx
|
||||
|
||||
content = f"**{title}**\n{message}"[:2000]
|
||||
resp = httpx.post(webhook_url, json={"content": content}, timeout=10.0)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
def _send_telegram(bot_token: str, chat_id: str, title: str, message: str) -> None:
|
||||
import httpx
|
||||
|
||||
from backend.webhooks import is_safe_target
|
||||
|
||||
url = f"https://api.telegram.org/bot{bot_token}/sendMessage"
|
||||
if not is_safe_target(url):
|
||||
raise RuntimeError("Cible Telegram bloquée par la politique SSRF")
|
||||
text = f"*{title}*\n{message}"[:4000]
|
||||
resp = httpx.post(
|
||||
url,
|
||||
json={"chat_id": chat_id, "text": text, "parse_mode": "Markdown"},
|
||||
timeout=10.0,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
def _send_smtp(config: dict[str, Any], password: str | None, title: str, message: str) -> None:
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = f"[ObsiGate] {title}"
|
||||
msg["From"] = config["from_addr"]
|
||||
msg["To"] = config["to_addr"]
|
||||
msg.set_content(message)
|
||||
with smtplib.SMTP(str(config["host"]), int(config.get("port", 587)), timeout=10) as client:
|
||||
if config.get("use_tls", True):
|
||||
client.starttls()
|
||||
if config.get("username") and password:
|
||||
client.login(str(config["username"]), password)
|
||||
client.send_message(msg)
|
||||
|
||||
|
||||
def _send_webhook(url: str, title: str, message: str, trigger: str) -> None:
|
||||
import httpx
|
||||
|
||||
from backend.webhooks import is_safe_target
|
||||
|
||||
if not is_safe_target(url):
|
||||
raise RuntimeError("Cible webhook bloquée par la politique SSRF")
|
||||
resp = httpx.post(
|
||||
url,
|
||||
json={
|
||||
"event": trigger,
|
||||
"title": title,
|
||||
"message": message,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"source": "obsigate-notify",
|
||||
},
|
||||
timeout=10.0,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
def send_via_channel(channel: dict[str, Any], title: str, message: str, trigger: str = "manual") -> None:
|
||||
"""Send a notification through one raw channel record. Raises on failure."""
|
||||
channel_type = channel.get("type")
|
||||
cfg = dict(channel.get("config", {}))
|
||||
secret = _get_secret(channel["id"])
|
||||
if channel_type == "discord":
|
||||
url = secret or cfg.get("webhook_url") or ""
|
||||
if not url or url == "***":
|
||||
raise RuntimeError("URL webhook Discord manquante")
|
||||
_send_discord(url, title, message)
|
||||
elif channel_type == "telegram":
|
||||
token = secret or cfg.get("bot_token") or os.environ.get("OBSIGATE_TELEGRAM_BOT_TOKEN") or ""
|
||||
if not token or token == "***":
|
||||
raise RuntimeError("Token bot Telegram manquant")
|
||||
_send_telegram(token, str(cfg.get("chat_id", "")), title, message)
|
||||
elif channel_type == "smtp":
|
||||
_send_smtp(cfg, secret, title, message)
|
||||
elif channel_type == "webhook":
|
||||
url = str(cfg.get("url") or "").strip()
|
||||
if not url:
|
||||
raise RuntimeError("URL webhook manquante")
|
||||
_send_webhook(url, title, message, trigger)
|
||||
else:
|
||||
raise RuntimeError(f"Type de canal inconnu : {channel_type}")
|
||||
|
||||
|
||||
def broadcast(trigger: str, title: str, message: str) -> list[dict[str, Any]]:
|
||||
"""Send to every enabled channel subscribed to *trigger*. Never raises."""
|
||||
results: list[dict[str, Any]] = []
|
||||
for channel in _read_channels():
|
||||
if not channel.get("enabled", True):
|
||||
continue
|
||||
if trigger not in channel.get("config", {}).get("triggers", ["manual"]):
|
||||
continue
|
||||
try:
|
||||
send_via_channel(channel, title, message, trigger)
|
||||
results.append({"channel_id": channel["id"], "ok": True})
|
||||
_mark_sent(channel["id"], None)
|
||||
except Exception as e:
|
||||
logger.warning(f"Notify channel '{channel.get('name')}' failed: {e}")
|
||||
results.append({"channel_id": channel["id"], "ok": False, "error": str(e)})
|
||||
_mark_sent(channel["id"], str(e))
|
||||
return results
|
||||
|
||||
|
||||
def _mark_sent(channel_id: str, error: str | None) -> None:
|
||||
with _lock:
|
||||
channels = _read_channels()
|
||||
for channel in channels:
|
||||
if channel.get("id") == channel_id:
|
||||
channel["last_sent_at"] = datetime.now(timezone.utc).isoformat()
|
||||
channel["last_error"] = error
|
||||
_write_channels(channels)
|
||||
@@ -41,6 +41,9 @@ TAGS_METADATA: list[dict[str, str]] = [
|
||||
{"name": "Admin", "description": "Admin-only system monitoring: stats, audit log, backup stats and live stream."},
|
||||
{"name": "Plugins", "description": "Install, enable and manage user plugins."},
|
||||
{"name": "Push", "description": "Web Push (VAPID) subscription management and test notifications."},
|
||||
{"name": "Duplicates", "description": "Duplicate-note detection and confirmed merge (#166)."},
|
||||
{"name": "Notify", "description": "External notifications: Discord, Telegram, SMTP and generic webhooks (#168)."},
|
||||
{"name": "Scheduler", "description": "Scheduled automatic tasks reusing the vault mutation services (#170)."},
|
||||
{"name": "Frontend", "description": "Static assets and SPA fallback routes."},
|
||||
]
|
||||
|
||||
@@ -95,6 +98,9 @@ _TAG_RULES: list[tuple[re.Pattern[str], str]] = [
|
||||
(re.compile(r"^/api/shares"), "Sharing"),
|
||||
(re.compile(r"^/s/"), "Sharing"),
|
||||
(re.compile(r"^/api/webhooks"), "Webhooks"),
|
||||
(re.compile(r"^/api/duplicates"), "Duplicates"),
|
||||
(re.compile(r"^/api/notify"), "Notify"),
|
||||
(re.compile(r"^/api/scheduler"), "Scheduler"),
|
||||
(re.compile(r"^/api/conflicts"), "Conflicts"),
|
||||
(re.compile(r"^/api/backups"), "Backups"),
|
||||
(re.compile(r"^/api/file/[^/]+/(backups|diff|restore)"), "Backups"),
|
||||
@@ -148,6 +154,9 @@ _TAG_ALIASES: dict[str, str] = {
|
||||
"export": "Export",
|
||||
"sharing": "Sharing",
|
||||
"webhooks": "Webhooks",
|
||||
"duplicates": "Duplicates",
|
||||
"notify": "Notify",
|
||||
"scheduler": "Scheduler",
|
||||
"conflicts": "Conflicts",
|
||||
"system": "System",
|
||||
"frontend": "Frontend",
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Duplicate detection & merge endpoints (#166).
|
||||
|
||||
Read endpoints require vault access; the merge endpoint is destructive
|
||||
(backup first in the service layer) and additionally requires the
|
||||
confirmation token pattern used by mutating routes — here enforced by an
|
||||
explicit ``confirm=true`` body flag, mirroring the agent two-step flow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.services import duplicates as _duplicates
|
||||
from backend.services.errors import ServiceError
|
||||
|
||||
router = APIRouter(prefix="/api/duplicates", tags=["duplicates"])
|
||||
|
||||
|
||||
class DuplicatePair(BaseModel):
|
||||
"""One candidate duplicate pair."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
file_a: str = Field(description="First file (vault-relative)")
|
||||
file_b: str = Field(description="Second file (vault-relative)")
|
||||
score: float = Field(description="Blended similarity in [0, 1]")
|
||||
|
||||
|
||||
class DuplicatesResponse(BaseModel):
|
||||
"""Response for GET /api/duplicates."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
vault: str = Field(description="Vault name")
|
||||
threshold: float = Field(description="Applied threshold")
|
||||
files_scanned: int = Field(description="Markdown files compared")
|
||||
truncated: bool = Field(description="True when the scan hit the file cap")
|
||||
pairs: list[DuplicatePair] = Field(description="Candidate pairs, best score first")
|
||||
|
||||
|
||||
class MergeResponse(BaseModel):
|
||||
"""Response for POST /api/duplicates/merge."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
strategy: str = Field(description="Applied merge strategy")
|
||||
target: str = Field(description="Surviving note")
|
||||
deleted: str = Field(description="Absorbed note (deleted after merge)")
|
||||
|
||||
|
||||
@router.get("", response_model=DuplicatesResponse)
|
||||
async def api_duplicates_list(
|
||||
vault: str = Query(..., description="Vault name"),
|
||||
threshold: float = Query(0.75, ge=0.3, le=1.0, description="Minimum similarity"),
|
||||
limit: int = Query(20, ge=1, le=200, description="Max pairs"),
|
||||
subdir: str = Query("", description="Directory scope"),
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""List candidate duplicate notes ordered by descending score."""
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(403, f"No access to vault '{vault}'")
|
||||
try:
|
||||
return _duplicates.find_duplicate_pairs(vault, threshold=threshold, limit=limit, subdir=subdir)
|
||||
except ServiceError as e:
|
||||
raise HTTPException(e.status or 400, e.message) from e
|
||||
|
||||
|
||||
@router.post("/merge", response_model=MergeResponse)
|
||||
async def api_duplicates_merge(
|
||||
body: dict[str, Any] = Body(...),
|
||||
current_user: dict[str, Any] = Depends(require_auth),
|
||||
):
|
||||
"""Merge *source_path* into *target_path* (``confirm: true`` required)."""
|
||||
vault = str(body.get("vault") or "")
|
||||
if not check_vault_access(vault, current_user):
|
||||
raise HTTPException(403, f"No access to vault '{vault}'")
|
||||
if body.get("confirm") is not True:
|
||||
raise HTTPException(400, "Fusion destructive : confirmez avec {confirm: true}")
|
||||
try:
|
||||
return _duplicates.merge_duplicates(
|
||||
vault,
|
||||
str(body.get("source_path") or ""),
|
||||
str(body.get("target_path") or ""),
|
||||
strategy=str(body.get("strategy") or "append"),
|
||||
)
|
||||
except ServiceError as e:
|
||||
raise HTTPException(e.status or 400, e.message) from e
|
||||
@@ -0,0 +1,96 @@
|
||||
"""External notification channels endpoints (#168).
|
||||
|
||||
Channel CRUD is admin-only (secrets involved); sending a test notification
|
||||
requires authentication. Responses mask secrets (``***`` + ``has_secret``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from backend import notify as _notify
|
||||
from backend.auth.middleware import require_admin, require_auth
|
||||
from backend.schemas import StatusResponse
|
||||
|
||||
router = APIRouter(prefix="/api/notify", tags=["notify"])
|
||||
|
||||
|
||||
class NotifyChannel(BaseModel):
|
||||
"""Public view of a notification channel (secrets masked)."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
id: str = Field(description="Channel id")
|
||||
name: str = Field(description="Display name")
|
||||
type: str = Field(description="discord | telegram | smtp | webhook")
|
||||
enabled: bool = Field(description="Whether the channel receives broadcasts")
|
||||
config: dict[str, Any] = Field(description="Channel config (secrets masked)")
|
||||
has_secret: bool = Field(description="True when a secret is configured")
|
||||
|
||||
|
||||
class NotifySendResult(BaseModel):
|
||||
"""Outcome of a test send / broadcast."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
ok: bool = Field(description="True when every delivery succeeded")
|
||||
deliveries: list[dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
@router.get("/channels", response_model=list[NotifyChannel])
|
||||
async def api_notify_list(current_user=Depends(require_admin)):
|
||||
"""List notification channels (admin)."""
|
||||
return _notify.list_channels()
|
||||
|
||||
|
||||
@router.post("/channels", response_model=NotifyChannel)
|
||||
async def api_notify_create(body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
"""Create a channel (``{name, type, config}``). Secrets go to the secret store."""
|
||||
try:
|
||||
return _notify.create_channel(
|
||||
str(body.get("name") or ""),
|
||||
str(body.get("type") or ""),
|
||||
dict(body.get("config") or {}),
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.patch("/channels/{channel_id}", response_model=NotifyChannel)
|
||||
async def api_notify_update(channel_id: str, body: dict = Body(...), current_user=Depends(require_admin)):
|
||||
"""Update a channel (name / enabled / config)."""
|
||||
try:
|
||||
result = _notify.update_channel(channel_id, body)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
if result is None:
|
||||
raise HTTPException(404, "Channel not found")
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/channels/{channel_id}", response_model=StatusResponse)
|
||||
async def api_notify_delete(channel_id: str, current_user=Depends(require_admin)):
|
||||
"""Delete a channel and its secret."""
|
||||
if not _notify.delete_channel(channel_id):
|
||||
raise HTTPException(404, "Channel not found")
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.post("/test", response_model=NotifySendResult)
|
||||
async def api_notify_test(body: dict = Body(...), current_user=Depends(require_auth)):
|
||||
"""Send a test notification (broadcast or single ``channel_id``)."""
|
||||
title = str(body.get("title") or "Test ObsiGate")
|
||||
message = str(body.get("message") or "Notification de test.")
|
||||
channel_id = str(body.get("channel_id") or "")
|
||||
if channel_id:
|
||||
channel = next((c for c in _notify._read_channels() if c.get("id") == channel_id), None)
|
||||
if channel is None:
|
||||
raise HTTPException(404, "Channel not found")
|
||||
try:
|
||||
_notify.send_via_channel(channel, title, message, "manual")
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"Envoi échoué : {e}") from e
|
||||
return {"ok": True, "deliveries": [{"channel_id": channel_id, "ok": True}]}
|
||||
deliveries = _notify.broadcast("manual", title, message)
|
||||
return {"ok": all(d.get("ok") for d in deliveries), "deliveries": deliveries}
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Scheduled tasks endpoints (#170).
|
||||
|
||||
Tasks reuse the existing mutation/notification services — this router only
|
||||
validates, persists and triggers. File-writing actions check vault access
|
||||
at creation time; the background tick re-checks nothing (system context) but
|
||||
records failures and notifies on ``schedule_failure`` (#168).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from backend import scheduler as _scheduler
|
||||
from backend.auth.middleware import check_vault_access, require_auth
|
||||
from backend.schemas import StatusResponse
|
||||
|
||||
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
|
||||
|
||||
|
||||
class ScheduledTask(BaseModel):
|
||||
"""A programmed automatic task."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
id: str = Field(description="Task id")
|
||||
name: str = Field(description="Display name")
|
||||
action: dict[str, Any] = Field(description="{kind, params}")
|
||||
schedule: dict[str, Any] = Field(description="{kind, ...}")
|
||||
enabled: bool = Field(description="Whether the tick executes it")
|
||||
created_by: str = Field(description="Owner username")
|
||||
created_at: str = Field(description="ISO-8601 creation time")
|
||||
last_run_at: str | None = Field(default=None)
|
||||
last_status: str | None = Field(default=None)
|
||||
last_error: str | None = Field(default=None)
|
||||
run_count: int = Field(default=0)
|
||||
next_run_at: str = Field(description="ISO-8601 next due time")
|
||||
|
||||
|
||||
class TaskRunResult(BaseModel):
|
||||
"""Outcome of a manual or due run."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
task_id: str = Field(description="Task id")
|
||||
ok: bool = Field(description="True on success")
|
||||
result: dict[str, Any] | None = Field(default=None)
|
||||
error: str | None = Field(default=None)
|
||||
|
||||
|
||||
def _check_action_vault(action: dict[str, Any], user: dict[str, Any]) -> None:
|
||||
from backend.services.errors import ServiceError
|
||||
from backend.services.vaults import get_vault_root
|
||||
|
||||
kind = (action or {}).get("kind")
|
||||
params = (action or {}).get("params") or {}
|
||||
if kind in ("create_file", "append_to_file"):
|
||||
vault = str(params.get("vault") or "")
|
||||
if not check_vault_access(vault, user):
|
||||
raise HTTPException(403, f"No access to vault '{vault}'")
|
||||
try:
|
||||
get_vault_root(vault)
|
||||
except ServiceError as e:
|
||||
raise HTTPException(404, f"Unknown vault '{vault}'") from e
|
||||
|
||||
|
||||
@router.get("/tasks", response_model=list[ScheduledTask])
|
||||
async def api_scheduler_list(current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""List scheduled tasks (newest first)."""
|
||||
return _scheduler.list_tasks()
|
||||
|
||||
|
||||
@router.post("/tasks", response_model=ScheduledTask)
|
||||
async def api_scheduler_create(body: dict = Body(...), current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Create a task (``{name, action, schedule, enabled?}``)."""
|
||||
action = dict(body.get("action") or {})
|
||||
_check_action_vault(action, current_user)
|
||||
try:
|
||||
return _scheduler.create_task(
|
||||
str(body.get("name") or ""),
|
||||
action,
|
||||
dict(body.get("schedule") or {}),
|
||||
created_by=str(current_user.get("username", "api")),
|
||||
enabled=bool(body.get("enabled", True)),
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
|
||||
|
||||
@router.patch("/tasks/{task_id}", response_model=ScheduledTask)
|
||||
async def api_scheduler_update(task_id: str, body: dict = Body(...), current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Update a task (name / enabled / action / schedule)."""
|
||||
if "action" in body:
|
||||
_check_action_vault(dict(body["action"] or {}), current_user)
|
||||
try:
|
||||
result = _scheduler.update_task(task_id, body)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
if result is None:
|
||||
raise HTTPException(404, "Task not found")
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/tasks/{task_id}", response_model=StatusResponse)
|
||||
async def api_scheduler_delete(task_id: str, current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Delete a task."""
|
||||
if not _scheduler.delete_task(task_id):
|
||||
raise HTTPException(404, "Task not found")
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.post("/tasks/{task_id}/run", response_model=TaskRunResult)
|
||||
async def api_scheduler_run(task_id: str, current_user: dict[str, Any] = Depends(require_auth)):
|
||||
"""Execute a task immediately (manual run)."""
|
||||
try:
|
||||
return _scheduler.run_task(task_id, manual=True)
|
||||
except KeyError:
|
||||
raise HTTPException(404, "Task not found") from None
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Scheduled tasks — automatic agent actions, type cron (#170).
|
||||
|
||||
Tasks are persisted in ``data/scheduled_tasks.json`` (guarded by an RLock,
|
||||
same pattern as the other JSON stores). Supported actions reuse the existing
|
||||
mutation/notification services — no new write path:
|
||||
|
||||
* ``create_file`` → ``backend.services.mutations.create_file``;
|
||||
* ``append_to_file`` → ``backend.services.mutations.append_to_file``;
|
||||
* ``notify`` → ``backend.notify.broadcast`` (trigger ``manual``).
|
||||
|
||||
Supported schedules:
|
||||
|
||||
* ``interval_hours`` — every N hours (N >= 0.25);
|
||||
* ``daily_time`` — once a day at ``HH:MM`` (local server time);
|
||||
* ``once_at`` — one shot at an ISO-8601 datetime (past = due immediately).
|
||||
|
||||
On failure the task records ``last_error`` and a ``schedule_failure``
|
||||
broadcast is emitted to the notification channels (#168) — best effort,
|
||||
never recursive (a failing ``notify`` action does not rebroadcast).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("obsigate.scheduler")
|
||||
|
||||
DATA_DIR = Path(os.environ.get("OBSIGATE_DATA_DIR", "data"))
|
||||
TASKS_FILE = DATA_DIR / "scheduled_tasks.json"
|
||||
|
||||
ACTION_KINDS = ("create_file", "append_to_file", "notify")
|
||||
SCHEDULE_KINDS = ("interval_hours", "daily_time", "once_at")
|
||||
|
||||
_lock = threading.RLock()
|
||||
|
||||
|
||||
# ── Store ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _read_tasks() -> list[dict[str, Any]]:
|
||||
if not TASKS_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(TASKS_FILE.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return []
|
||||
|
||||
|
||||
def _write_tasks(tasks: list[dict[str, Any]]) -> None:
|
||||
TASKS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = TASKS_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(tasks, indent=2, default=str), encoding="utf-8")
|
||||
tmp.replace(TASKS_FILE)
|
||||
|
||||
|
||||
def list_tasks() -> list[dict[str, Any]]:
|
||||
"""Return all scheduled tasks (newest first)."""
|
||||
return sorted(_read_tasks(), key=lambda t: t.get("created_at", ""), reverse=True)
|
||||
|
||||
|
||||
def get_task(task_id: str) -> dict[str, Any] | None:
|
||||
"""Return one task by id, or None."""
|
||||
for task in _read_tasks():
|
||||
if task.get("id") == task_id:
|
||||
return task
|
||||
return None
|
||||
|
||||
|
||||
# ── Validation ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _validate_action(action: dict[str, Any]) -> dict[str, Any]:
|
||||
kind = action.get("kind")
|
||||
if kind not in ACTION_KINDS:
|
||||
raise ValueError(f"Action inconnue : {kind} (attendu : {', '.join(ACTION_KINDS)})")
|
||||
params = dict(action.get("params") or {})
|
||||
if kind in ("create_file", "append_to_file"):
|
||||
if not str(params.get("vault") or "").strip():
|
||||
raise ValueError("params.vault requis pour create_file/append_to_file")
|
||||
if not str(params.get("path") or "").strip():
|
||||
raise ValueError("params.path requis pour create_file/append_to_file")
|
||||
if kind == "append_to_file" and not str(params.get("content") or ""):
|
||||
raise ValueError("params.content requis pour append_to_file")
|
||||
elif kind == "notify":
|
||||
if not str(params.get("title") or "").strip():
|
||||
raise ValueError("params.title requis pour notify")
|
||||
if not str(params.get("message") or "").strip():
|
||||
raise ValueError("params.message requis pour notify")
|
||||
return {"kind": kind, "params": params}
|
||||
|
||||
|
||||
def _validate_schedule(schedule: dict[str, Any]) -> dict[str, Any]:
|
||||
kind = schedule.get("kind")
|
||||
if kind not in SCHEDULE_KINDS:
|
||||
raise ValueError(f"Planification inconnue : {kind} (attendu : {', '.join(SCHEDULE_KINDS)})")
|
||||
if kind == "interval_hours":
|
||||
hours = float(schedule.get("hours") or 0)
|
||||
if hours < 0.25:
|
||||
raise ValueError("hours doit être >= 0.25")
|
||||
return {"kind": kind, "hours": hours}
|
||||
if kind == "daily_time":
|
||||
at = str(schedule.get("at") or "").strip()
|
||||
try:
|
||||
datetime.strptime(at, "%H:%M")
|
||||
except ValueError:
|
||||
raise ValueError("at doit être au format HH:MM (ex. 08:30)") from None
|
||||
return {"kind": kind, "at": at}
|
||||
# once_at
|
||||
at = str(schedule.get("at") or "").strip()
|
||||
try:
|
||||
parsed = datetime.fromisoformat(at)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
raise ValueError("at doit être une date ISO-8601 (ex. 2026-10-05T08:30:00)") from None
|
||||
return {"kind": kind, "at": parsed.isoformat()}
|
||||
|
||||
|
||||
# ── CRUD ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def create_task(
|
||||
name: str,
|
||||
action: dict[str, Any],
|
||||
schedule: dict[str, Any],
|
||||
*,
|
||||
created_by: str = "api",
|
||||
enabled: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Create a scheduled task. Raises ValueError on invalid action/schedule."""
|
||||
validated_action = _validate_action(action)
|
||||
validated_schedule = _validate_schedule(schedule)
|
||||
now = datetime.now(timezone.utc)
|
||||
with _lock:
|
||||
tasks = _read_tasks()
|
||||
task = {
|
||||
"id": str(uuid.uuid4()),
|
||||
"name": (name or validated_action["kind"]).strip() or validated_action["kind"],
|
||||
"action": validated_action,
|
||||
"schedule": validated_schedule,
|
||||
"enabled": bool(enabled),
|
||||
"created_by": created_by,
|
||||
"created_at": now.isoformat(),
|
||||
"last_run_at": None,
|
||||
"last_status": None,
|
||||
"last_error": None,
|
||||
"run_count": 0,
|
||||
"next_run_at": compute_next_run(
|
||||
{"schedule": validated_schedule, "last_run_at": None}, now
|
||||
).isoformat(),
|
||||
}
|
||||
tasks.append(task)
|
||||
_write_tasks(tasks)
|
||||
logger.info(f"Scheduled task created: '{task['name']}' ({validated_schedule['kind']})")
|
||||
return task
|
||||
|
||||
|
||||
def update_task(task_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Update name/enabled/action/schedule. Returns None when unknown."""
|
||||
with _lock:
|
||||
tasks = _read_tasks()
|
||||
for task in tasks:
|
||||
if task.get("id") != task_id:
|
||||
continue
|
||||
if updates.get("name"):
|
||||
task["name"] = str(updates["name"])
|
||||
if "enabled" in updates:
|
||||
task["enabled"] = bool(updates["enabled"])
|
||||
if "action" in updates:
|
||||
task["action"] = _validate_action(updates["action"])
|
||||
if "schedule" in updates:
|
||||
task["schedule"] = _validate_schedule(updates["schedule"])
|
||||
task["next_run_at"] = compute_next_run(task).isoformat()
|
||||
_write_tasks(tasks)
|
||||
return task
|
||||
return None
|
||||
|
||||
|
||||
def delete_task(task_id: str) -> bool:
|
||||
"""Delete a task. Returns False when unknown."""
|
||||
with _lock:
|
||||
tasks = _read_tasks()
|
||||
remaining = [t for t in tasks if t.get("id") != task_id]
|
||||
if len(remaining) == len(tasks):
|
||||
return False
|
||||
_write_tasks(remaining)
|
||||
return True
|
||||
|
||||
|
||||
# ── Scheduling ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def compute_next_run(task: dict[str, Any], now: datetime | None = None) -> datetime:
|
||||
"""Compute the next due datetime for *task*."""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
if now.tzinfo is None:
|
||||
now = now.replace(tzinfo=timezone.utc)
|
||||
schedule = task.get("schedule", {})
|
||||
kind = schedule.get("kind")
|
||||
last_run_at = task.get("last_run_at")
|
||||
last = None
|
||||
if last_run_at:
|
||||
try:
|
||||
last = datetime.fromisoformat(str(last_run_at))
|
||||
if last.tzinfo is None:
|
||||
last = last.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
last = None
|
||||
if kind == "interval_hours":
|
||||
hours = float(schedule.get("hours", 24))
|
||||
base = last or now
|
||||
nxt = base + timedelta(hours=hours)
|
||||
# Première planification : due dès maintenant + intervalle ? Non —
|
||||
# la tâche démarre au prochain intervalle, sauf retard déjà accumulé.
|
||||
if last is None:
|
||||
nxt = now + timedelta(hours=hours)
|
||||
return max(now, nxt)
|
||||
if kind == "daily_time":
|
||||
hour, minute = (str(schedule.get("at", "08:00")) + ":00").split(":")[:2]
|
||||
candidate = now.replace(hour=int(hour), minute=int(minute), second=0, microsecond=0)
|
||||
if candidate <= now:
|
||||
candidate += timedelta(days=1)
|
||||
return candidate
|
||||
if kind == "once_at":
|
||||
try:
|
||||
at = datetime.fromisoformat(str(schedule.get("at")))
|
||||
if at.tzinfo is None:
|
||||
at = at.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
return now
|
||||
if task.get("last_run_at"):
|
||||
return datetime.max.replace(tzinfo=timezone.utc) # déjà exécutée
|
||||
return at
|
||||
return now + timedelta(hours=24)
|
||||
|
||||
|
||||
def _execute_action(task: dict[str, Any]) -> dict[str, Any]:
|
||||
action = task["action"]
|
||||
kind = action["kind"]
|
||||
params = action["params"]
|
||||
if kind == "create_file":
|
||||
from backend.services.mutations import create_file
|
||||
|
||||
return create_file(
|
||||
params["vault"],
|
||||
params["path"],
|
||||
params.get("content", ""),
|
||||
overwrite=bool(params.get("overwrite", False)),
|
||||
)
|
||||
if kind == "append_to_file":
|
||||
from backend.services.mutations import append_to_file
|
||||
|
||||
return append_to_file(params["vault"], params["path"], params.get("content", ""))
|
||||
if kind == "notify":
|
||||
from backend.notify import broadcast
|
||||
|
||||
results = broadcast("manual", str(params["title"]), str(params.get("message", "")))
|
||||
return {"broadcast": results}
|
||||
raise ValueError(f"Action inconnue : {kind}")
|
||||
|
||||
|
||||
def run_task(task_id: str, *, manual: bool = False) -> dict[str, Any]:
|
||||
"""Execute one task now (manual or due). Records status; notifies on failure."""
|
||||
with _lock:
|
||||
tasks = _read_tasks()
|
||||
task = next((t for t in tasks if t.get("id") == task_id), None)
|
||||
if task is None:
|
||||
raise KeyError(task_id)
|
||||
if not task.get("enabled", True) and not manual:
|
||||
return {"task_id": task_id, "skipped": True, "reason": "disabled"}
|
||||
try:
|
||||
result = _execute_action(task)
|
||||
task["last_run_at"] = datetime.now(timezone.utc).isoformat()
|
||||
task["last_status"] = "ok"
|
||||
task["last_error"] = None
|
||||
task["run_count"] = int(task.get("run_count", 0)) + 1
|
||||
if task.get("schedule", {}).get("kind") == "once_at":
|
||||
task["enabled"] = False # one-shot consommé
|
||||
task["next_run_at"] = compute_next_run(task).isoformat()
|
||||
_write_tasks(tasks)
|
||||
if manual:
|
||||
from backend.notify import broadcast
|
||||
|
||||
broadcast("schedule_success", f"Tâche « {task['name']} » OK", "Exécution manuelle réussie.")
|
||||
return {"task_id": task_id, "ok": True, "result": result}
|
||||
except Exception as e:
|
||||
task["last_run_at"] = datetime.now(timezone.utc).isoformat()
|
||||
task["last_status"] = "error"
|
||||
task["last_error"] = str(e)
|
||||
task["run_count"] = int(task.get("run_count", 0)) + 1
|
||||
task["next_run_at"] = compute_next_run(task).isoformat()
|
||||
_write_tasks(tasks)
|
||||
logger.warning(f"Scheduled task '{task.get('name')}' failed: {e}")
|
||||
if task["action"]["kind"] != "notify":
|
||||
try:
|
||||
from backend.notify import broadcast
|
||||
|
||||
broadcast(
|
||||
"schedule_failure",
|
||||
f"Échec tâche « {task.get('name')} »",
|
||||
f"{e}",
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failure notification broadcast failed", exc_info=True)
|
||||
return {"task_id": task_id, "ok": False, "error": str(e)}
|
||||
|
||||
|
||||
def tick(now: datetime | None = None) -> list[dict[str, Any]]:
|
||||
"""Run every due task. Returns per-task outcomes (empty when idle)."""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
outcomes: list[dict[str, Any]] = []
|
||||
for task in _read_tasks():
|
||||
if not task.get("enabled", True):
|
||||
continue
|
||||
try:
|
||||
next_run = datetime.fromisoformat(str(task.get("next_run_at") or ""))
|
||||
if next_run.tzinfo is None:
|
||||
next_run = next_run.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
next_run = compute_next_run(task, now)
|
||||
if next_run <= now:
|
||||
outcomes.append(run_task(task["id"]))
|
||||
return outcomes
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Duplicate detection & merge services (#166).
|
||||
|
||||
Single source of truth consumed by the REST routes
|
||||
(``/api/duplicates``) and the AI tool layer (``find_duplicates``,
|
||||
``merge_duplicate_notes``).
|
||||
|
||||
Method is deterministic stdlib-only: frontmatter stripped, token-set
|
||||
Jaccard blended with a title similarity. No embedding dependency —
|
||||
the semantic index (#70) stays an optional refinement, not a requirement.
|
||||
|
||||
Fusion never runs without an explicit confirmation: the tool layer
|
||||
registers the merge as ``DANGEROUS`` (two-step propose/apply) and this
|
||||
service takes an automatic backup before any destructive write.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.services.backups import create_backup
|
||||
from backend.services.errors import ServiceError
|
||||
from backend.services.paths import resolve_safe_path
|
||||
from backend.services.vaults import get_vault_root
|
||||
|
||||
logger = logging.getLogger("obsigate.services.duplicates")
|
||||
|
||||
MAX_FILES_SCANNED = 500
|
||||
MAX_FILE_BYTES = 200_000
|
||||
MAX_CONTENT_CHARS = 50_000
|
||||
|
||||
_WORD_RE = re.compile(r"[\w]+", re.UNICODE)
|
||||
_FRONTMATTER_RE = re.compile(r"\A---\s*\n.*?\n---\s*\n", re.DOTALL)
|
||||
|
||||
|
||||
def _strip_frontmatter(text: str) -> str:
|
||||
"""Remove a leading YAML frontmatter block, if present."""
|
||||
return _FRONTMATTER_RE.sub("", text, count=1)
|
||||
|
||||
|
||||
def _tokens(text: str) -> set[str]:
|
||||
"""Lowercase word tokens (keeps accents), stop-words free but tiny tokens dropped."""
|
||||
return {t for t in _WORD_RE.findall(text.lower()) if len(t) > 2}
|
||||
|
||||
|
||||
def similarity_score(a: str, b: str) -> float:
|
||||
"""Blend Jaccard (0.7) + title/first-line similarity (0.3) in [0, 1].
|
||||
|
||||
Pure function — unit-tested directly.
|
||||
"""
|
||||
ta, tb = _tokens(_strip_frontmatter(a)), _tokens(_strip_frontmatter(b))
|
||||
if not ta or not tb:
|
||||
return 0.0
|
||||
jaccard = len(ta & tb) / len(ta | tb)
|
||||
head_a = (a.strip().splitlines() or [""])[:1][0][:200].lower()
|
||||
head_b = (b.strip().splitlines() or [""])[:1][0][:200].lower()
|
||||
title_sim = SequenceMatcher(None, head_a, head_b).ratio() if head_a and head_b else 0.0
|
||||
return round(0.7 * jaccard + 0.3 * title_sim, 4)
|
||||
|
||||
|
||||
def _iter_markdown_files(root: Path, subdir: str = "") -> list[Path]:
|
||||
base = resolve_safe_path(root, subdir) if subdir else root.resolve()
|
||||
if not base.exists() or not base.is_dir():
|
||||
raise ServiceError(
|
||||
f"Directory not found: {subdir or '.'}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"path": subdir},
|
||||
)
|
||||
files = sorted(
|
||||
(p for p in base.rglob("*.md") if p.is_file() and not p.is_symlink()),
|
||||
key=lambda p: str(p),
|
||||
)
|
||||
return files[:MAX_FILES_SCANNED]
|
||||
|
||||
|
||||
def _read_capped(path: Path) -> str:
|
||||
try:
|
||||
if path.stat().st_size > MAX_FILE_BYTES:
|
||||
return ""
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return ""
|
||||
return text[:MAX_CONTENT_CHARS]
|
||||
|
||||
|
||||
def find_duplicate_pairs(
|
||||
vault: str,
|
||||
threshold: float = 0.75,
|
||||
limit: int = 50,
|
||||
subdir: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""Return candidate duplicate pairs ordered by descending score.
|
||||
|
||||
Args:
|
||||
vault: Vault name.
|
||||
threshold: Minimum blended score in [0.3, 1.0].
|
||||
limit: Max pairs returned (1-200).
|
||||
subdir: Optional vault-relative directory scope.
|
||||
"""
|
||||
if not 0.3 <= threshold <= 1.0:
|
||||
raise ServiceError(
|
||||
"threshold must be between 0.3 and 1.0",
|
||||
code="invalid_arguments",
|
||||
status=400,
|
||||
)
|
||||
limit = max(1, min(limit, 200))
|
||||
root = get_vault_root(vault)
|
||||
files = _iter_markdown_files(root, subdir)
|
||||
contents: dict[str, str] = {}
|
||||
token_sets: dict[str, set[str]] = {}
|
||||
for path in files:
|
||||
rel = str(path.relative_to(root)).replace("\\", "/")
|
||||
text = _read_capped(path)
|
||||
if not text.strip():
|
||||
continue
|
||||
contents[rel] = text
|
||||
token_sets[rel] = _tokens(_strip_frontmatter(text))
|
||||
|
||||
rels = sorted(contents)
|
||||
pairs: list[dict[str, Any]] = []
|
||||
for i in range(len(rels)):
|
||||
for j in range(i + 1, len(rels)):
|
||||
a, b = rels[i], rels[j]
|
||||
ta, tb = token_sets[a], token_sets[b]
|
||||
if not ta or not tb:
|
||||
continue
|
||||
# Cheap pre-filter: Jaccard lower bound before the full score.
|
||||
inter = len(ta & tb)
|
||||
union = len(ta | tb)
|
||||
if union == 0 or inter / union < threshold * 0.6:
|
||||
continue
|
||||
score = similarity_score(contents[a], contents[b])
|
||||
if score >= threshold:
|
||||
pairs.append({"file_a": a, "file_b": b, "score": score})
|
||||
pairs.sort(key=lambda p: p["score"], reverse=True)
|
||||
return {
|
||||
"vault": vault,
|
||||
"threshold": threshold,
|
||||
"files_scanned": len(contents),
|
||||
"truncated": len(files) >= MAX_FILES_SCANNED,
|
||||
"pairs": pairs[:limit],
|
||||
}
|
||||
|
||||
|
||||
def merge_duplicates(
|
||||
vault: str,
|
||||
source_path: str,
|
||||
target_path: str,
|
||||
strategy: str = "append",
|
||||
) -> dict[str, Any]:
|
||||
"""Merge *source_path* into *target_path*, then delete the source.
|
||||
|
||||
Strategies:
|
||||
``append`` — source content appended after target (separator + origin
|
||||
marker), source deleted.
|
||||
``prefer_target`` — source deleted, target untouched (dedupe only).
|
||||
``prefer_source`` — target overwritten with source content, source deleted.
|
||||
|
||||
A backup of both files is taken first; the source deletion also goes
|
||||
through the backup-aware mutation service.
|
||||
"""
|
||||
from backend.services import mutations as _mutations
|
||||
|
||||
if strategy not in ("append", "prefer_target", "prefer_source"):
|
||||
raise ServiceError(
|
||||
f"Unknown strategy: {strategy}",
|
||||
code="invalid_arguments",
|
||||
status=400,
|
||||
)
|
||||
if source_path == target_path:
|
||||
raise ServiceError(
|
||||
"source_path and target_path must differ",
|
||||
code="invalid_arguments",
|
||||
status=400,
|
||||
)
|
||||
root = get_vault_root(vault)
|
||||
src = resolve_safe_path(root, source_path)
|
||||
dst = resolve_safe_path(root, target_path)
|
||||
if not src.is_file() or src.suffix.lower() != ".md":
|
||||
raise ServiceError(
|
||||
f"Source not found: {source_path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"path": source_path},
|
||||
)
|
||||
if not dst.is_file() or dst.suffix.lower() != ".md":
|
||||
raise ServiceError(
|
||||
f"Target not found: {target_path}",
|
||||
code="not_found",
|
||||
status=404,
|
||||
details={"path": target_path},
|
||||
)
|
||||
# Backup préalable (jamais de fusion sans filet — critère #166).
|
||||
create_backup(src, vault, source_path)
|
||||
create_backup(dst, vault, target_path)
|
||||
|
||||
if strategy == "prefer_target":
|
||||
result = _mutations.delete_file(vault, source_path)
|
||||
return {"strategy": strategy, "target": target_path, "deleted": source_path, "delete": result}
|
||||
if strategy == "prefer_source":
|
||||
content = src.read_text(encoding="utf-8", errors="replace")
|
||||
result = _mutations.edit_file(vault, target_path, content)
|
||||
deleted = _mutations.delete_file(vault, source_path)
|
||||
return {"strategy": strategy, "target": target_path, "edit": result, "deleted": source_path, "delete": deleted}
|
||||
# append
|
||||
target_text = dst.read_text(encoding="utf-8", errors="replace")
|
||||
source_text = src.read_text(encoding="utf-8", errors="replace")
|
||||
merged = target_text.rstrip() + f"\n\n---\n\n_Fusionné depuis `{source_path}` (#166)_\n\n" + source_text.lstrip()
|
||||
result = _mutations.edit_file(vault, target_path, merged)
|
||||
deleted = _mutations.delete_file(vault, source_path)
|
||||
return {"strategy": strategy, "target": target_path, "edit": result, "deleted": source_path, "delete": deleted}
|
||||
@@ -12,6 +12,9 @@ which ``.gitignore`` excludes via ``_*.py``), hence this explicit facade.
|
||||
from backend.tools import connected as _connected # noqa: F401 (registers connected-source tools)
|
||||
from backend.tools import crawler as _crawler # noqa: F401 (registers the site crawler)
|
||||
from backend.tools import documents as _documents # noqa: F401 (registers document tools)
|
||||
from backend.tools import duplicates as _duplicates # noqa: F401 (registers duplicate tools #166)
|
||||
from backend.tools import notify as _notify_tools # noqa: F401 (registers notify tool #168)
|
||||
from backend.tools import scheduled as _scheduled # noqa: F401 (registers scheduler tools #170)
|
||||
from backend.tools import service as _service # noqa: F401 (registers tools)
|
||||
from backend.tools import spreadsheets as _spreadsheets # noqa: F401 (registers existing-workbook tools #153 A6)
|
||||
from backend.tools import web as _web # noqa: F401 (registers web tools)
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Duplicate detection & merge tools (#166).
|
||||
|
||||
* ``find_duplicates`` — READ, vault-scoped: candidate pairs with scores.
|
||||
* ``merge_duplicate_notes`` — DANGEROUS: confirmed fusion with automatic
|
||||
backup (service layer), never without an explicit approval.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from backend.services import duplicates as _duplicates
|
||||
from backend.services.errors import ServiceError
|
||||
from backend.tools.context import ToolContext, ToolError, ToolRisk
|
||||
from backend.tools.registry import tool
|
||||
from backend.tools.schemas import FindDuplicatesInput, MergeDuplicatesInput
|
||||
|
||||
|
||||
@tool(
|
||||
name="find_duplicates",
|
||||
description="Find candidate duplicate markdown notes in a vault (similarity scores).",
|
||||
input_model=FindDuplicatesInput,
|
||||
risk=ToolRisk.READ,
|
||||
requires_vault=True,
|
||||
)
|
||||
def find_duplicates(ctx: ToolContext, params: FindDuplicatesInput) -> dict[str, Any]:
|
||||
"""List duplicate candidates ordered by descending score."""
|
||||
try:
|
||||
return _duplicates.find_duplicate_pairs(
|
||||
params.vault,
|
||||
threshold=params.threshold,
|
||||
limit=params.limit,
|
||||
subdir=params.subdir,
|
||||
)
|
||||
except ServiceError as e:
|
||||
raise ToolError(e.message, code=e.code, details=e.details) from e
|
||||
|
||||
|
||||
@tool(
|
||||
name="merge_duplicate_notes",
|
||||
description=(
|
||||
"Merge one note into another and delete the source (backup first). "
|
||||
"Destructive: requires confirmation."
|
||||
),
|
||||
input_model=MergeDuplicatesInput,
|
||||
risk=ToolRisk.DANGEROUS,
|
||||
requires_vault=True,
|
||||
)
|
||||
def merge_duplicate_notes(ctx: ToolContext, params: MergeDuplicatesInput) -> dict[str, Any]:
|
||||
"""Fuse *source_path* into *target_path* using the chosen strategy."""
|
||||
try:
|
||||
return _duplicates.merge_duplicates(
|
||||
params.vault,
|
||||
params.source_path,
|
||||
params.target_path,
|
||||
strategy=params.strategy,
|
||||
)
|
||||
except ServiceError as e:
|
||||
raise ToolError(e.message, code=e.code, details=e.details) from e
|
||||
@@ -62,6 +62,13 @@ _STEP_LABELS: dict[str, tuple[str, str | None]] = {
|
||||
"create_docx": ("docx_create", "path"),
|
||||
"create_csv": ("csv_create", "path"),
|
||||
"create_pdf": ("pdf_create", "path"),
|
||||
"find_duplicates": ("duplicates", "vault"),
|
||||
"merge_duplicate_notes": ("duplicates_merge", "source_path"),
|
||||
"notify_external": ("notify", "title"),
|
||||
"create_scheduled_task": ("schedule_create", "name"),
|
||||
"list_scheduled_tasks": ("schedule_list", None),
|
||||
"delete_scheduled_task": ("schedule_delete", "task_id"),
|
||||
"run_scheduled_task_now": ("schedule_run", "task_id"),
|
||||
}
|
||||
|
||||
GENERIC_KEY = "generic"
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""External notification tool (#168) — Discord, Telegram, SMTP, webhook.
|
||||
|
||||
``notify_external`` is WRITE (external side effect → confirmation card in the
|
||||
UI, propose/apply over MCP). Delivery itself lives in :mod:`backend.notify`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from backend import notify as _notify
|
||||
from backend.tools.context import ToolContext, ToolError, ToolRisk
|
||||
from backend.tools.registry import tool
|
||||
from backend.tools.schemas import NotifyExternalInput
|
||||
|
||||
|
||||
@tool(
|
||||
name="notify_external",
|
||||
description="Send a notification through external channels (Discord, Telegram, SMTP, webhook).",
|
||||
input_model=NotifyExternalInput,
|
||||
risk=ToolRisk.WRITE,
|
||||
)
|
||||
def notify_external(ctx: ToolContext, params: NotifyExternalInput) -> dict[str, Any]:
|
||||
"""Broadcast to the trigger scope, or target a single channel id."""
|
||||
try:
|
||||
if params.channel_id:
|
||||
channel = next(
|
||||
(c for c in _notify._read_channels() if c.get("id") == params.channel_id),
|
||||
None,
|
||||
)
|
||||
if channel is None:
|
||||
raise ToolError(f"Unknown channel: {params.channel_id}", code="not_found")
|
||||
if not channel.get("enabled", True):
|
||||
raise ToolError(f"Channel disabled: {params.channel_id}", code="invalid_arguments")
|
||||
_notify.send_via_channel(channel, params.title, params.message, params.trigger)
|
||||
return {"ok": True, "channel_id": params.channel_id}
|
||||
results = _notify.broadcast(params.trigger, params.title, params.message)
|
||||
return {"ok": True, "deliveries": results}
|
||||
except ToolError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise ToolError(f"Notification failed: {e}", code="notify_failed") from e
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Scheduled-task tools (#170) — the agent programs its own cron.
|
||||
|
||||
* ``create_scheduled_task`` — WRITE (a future write, confirmed once now).
|
||||
* ``list_scheduled_tasks`` — READ.
|
||||
* ``delete_scheduled_task`` — WRITE (removes a future side effect).
|
||||
* ``run_scheduled_task_now`` — WRITE (immediate side effect).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from backend import scheduler as _scheduler
|
||||
from backend.tools.context import ToolContext, ToolError, ToolRisk
|
||||
from backend.tools.registry import tool
|
||||
from backend.tools.schemas import (
|
||||
CreateScheduledTaskInput,
|
||||
DeleteScheduledTaskInput,
|
||||
ListVaultsInput,
|
||||
RunScheduledTaskInput,
|
||||
)
|
||||
|
||||
|
||||
@tool(
|
||||
name="create_scheduled_task",
|
||||
description="Program an automatic task (create_file, append_to_file, notify) on a cron-like schedule.",
|
||||
input_model=CreateScheduledTaskInput,
|
||||
risk=ToolRisk.WRITE,
|
||||
)
|
||||
def create_scheduled_task(ctx: ToolContext, params: CreateScheduledTaskInput) -> dict[str, Any]:
|
||||
"""Create a task owned by the requesting user."""
|
||||
try:
|
||||
return _scheduler.create_task(
|
||||
params.name,
|
||||
params.action,
|
||||
params.schedule,
|
||||
created_by=ctx.username,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise ToolError(str(e), code="invalid_arguments") from e
|
||||
|
||||
|
||||
@tool(
|
||||
name="list_scheduled_tasks",
|
||||
description="List automatic tasks programmed in ObsiGate.",
|
||||
input_model=ListVaultsInput,
|
||||
risk=ToolRisk.READ,
|
||||
)
|
||||
def list_scheduled_tasks(ctx: ToolContext, _params: ListVaultsInput) -> list[dict[str, Any]]:
|
||||
"""Return tasks newest first."""
|
||||
return _scheduler.list_tasks()
|
||||
|
||||
|
||||
@tool(
|
||||
name="delete_scheduled_task",
|
||||
description="Delete a programmed automatic task.",
|
||||
input_model=DeleteScheduledTaskInput,
|
||||
risk=ToolRisk.WRITE,
|
||||
)
|
||||
def delete_scheduled_task(ctx: ToolContext, params: DeleteScheduledTaskInput) -> dict[str, Any]:
|
||||
"""Delete by id; unknown id is a not_found tool error."""
|
||||
if not _scheduler.delete_task(params.task_id):
|
||||
raise ToolError(f"Unknown task: {params.task_id}", code="not_found")
|
||||
return {"ok": True, "task_id": params.task_id}
|
||||
|
||||
|
||||
@tool(
|
||||
name="run_scheduled_task_now",
|
||||
description="Execute a programmed task immediately (manual run).",
|
||||
input_model=RunScheduledTaskInput,
|
||||
risk=ToolRisk.WRITE,
|
||||
)
|
||||
def run_scheduled_task_now(ctx: ToolContext, params: RunScheduledTaskInput) -> dict[str, Any]:
|
||||
"""Run now and return the outcome (failures are recorded + notified)."""
|
||||
try:
|
||||
return _scheduler.run_task(params.task_id, manual=True)
|
||||
except KeyError as e:
|
||||
raise ToolError(f"Unknown task: {params.task_id}", code="not_found") from e
|
||||
@@ -441,6 +441,53 @@ class PdfInput(BaseModel):
|
||||
overwrite: bool = Field(True, description="Replace an existing file (with backup)")
|
||||
|
||||
|
||||
class FindDuplicatesInput(BaseModel):
|
||||
"""Find candidate duplicate notes in a vault (#166)."""
|
||||
|
||||
vault: str = Field(..., description="Vault name")
|
||||
threshold: float = Field(0.75, ge=0.3, le=1.0, description="Minimum similarity score")
|
||||
limit: int = Field(20, ge=1, le=200, description="Maximum number of pairs")
|
||||
subdir: str = Field("", description="Vault-relative directory scope (empty = whole vault)")
|
||||
|
||||
|
||||
class MergeDuplicatesInput(BaseModel):
|
||||
"""Merge one note into another, then delete the source (#166, destructive)."""
|
||||
|
||||
vault: str = Field(..., description="Vault name")
|
||||
source_path: str = Field(..., description="Vault-relative path of the note to absorb")
|
||||
target_path: str = Field(..., description="Vault-relative path of the surviving note")
|
||||
strategy: str = Field("append", description="'append', 'prefer_target' or 'prefer_source'")
|
||||
|
||||
|
||||
class NotifyExternalInput(BaseModel):
|
||||
"""Send a notification through external channels (#168)."""
|
||||
|
||||
title: str = Field(..., min_length=1, description="Notification title")
|
||||
message: str = Field(..., min_length=1, description="Notification body")
|
||||
trigger: str = Field("manual", description="Trigger scope: manual, schedule_failure, schedule_success")
|
||||
channel_id: str = Field("", description="Single channel id (empty = broadcast to trigger)")
|
||||
|
||||
|
||||
class CreateScheduledTaskInput(BaseModel):
|
||||
"""Create an automatic task executed by the scheduler (#170)."""
|
||||
|
||||
name: str = Field(..., min_length=1, description="Task display name")
|
||||
action: dict[str, Any] = Field(..., description="{kind, params} (create_file, append_to_file, notify)")
|
||||
schedule: dict[str, Any] = Field(..., description="{kind, ...} (interval_hours, daily_time, once_at)")
|
||||
|
||||
|
||||
class DeleteScheduledTaskInput(BaseModel):
|
||||
"""Delete a scheduled task by id (#170)."""
|
||||
|
||||
task_id: str = Field(..., min_length=1, description="Task id")
|
||||
|
||||
|
||||
class RunScheduledTaskInput(BaseModel):
|
||||
"""Execute a scheduled task immediately (#170)."""
|
||||
|
||||
task_id: str = Field(..., min_length=1, description="Task id")
|
||||
|
||||
|
||||
class ToolResult(BaseModel):
|
||||
"""Uniform result returned by :func:`backend.tools.registry.call_tool`."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user