139 lines
4.4 KiB
Python
139 lines
4.4 KiB
Python
"""SQLite cache for web tool results (search results, fetched pages).
|
|
|
|
Phase 2 of the web-toolset roadmap (« Transverse »): repeated web searches and
|
|
page fetches (common in agent loops, where the model re-reads a source) must
|
|
not hammer the providers. Results are cached in a dedicated SQLite table with
|
|
a TTL; the cache is best-effort — any error silently disables it so a broken
|
|
database file never takes the assistant down.
|
|
|
|
Configuration (environment):
|
|
* ``OBSIGATE_DATA_DIR`` — base data directory (default ``data``)
|
|
* ``OBSIGATE_WEB_CACHE_PATH`` — explicit cache file override
|
|
* ``OBSIGATE_WEB_CACHE_TTL`` — seconds, ``0`` disables the cache (default 900)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger("obsigate.tools.webcache")
|
|
|
|
DEFAULT_TTL_SECONDS = 900
|
|
_schema_ready = False
|
|
_write_lock = threading.Lock()
|
|
|
|
|
|
def ttl_seconds() -> float:
|
|
"""Configured TTL in seconds (``0`` = cache disabled)."""
|
|
return float(os.environ.get("OBSIGATE_WEB_CACHE_TTL", str(DEFAULT_TTL_SECONDS)))
|
|
|
|
|
|
def _cache_path() -> Path:
|
|
override = os.environ.get("OBSIGATE_WEB_CACHE_PATH", "").strip()
|
|
if override:
|
|
return Path(override)
|
|
return Path(os.environ.get("OBSIGATE_DATA_DIR", "data")) / "web_cache.sqlite3"
|
|
|
|
|
|
def _connect() -> sqlite3.Connection:
|
|
"""Open (and lazily create) the cache database."""
|
|
global _schema_ready
|
|
path = _cache_path()
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
conn = sqlite3.connect(path, timeout=5, check_same_thread=False)
|
|
if not _schema_ready:
|
|
conn.execute(
|
|
"CREATE TABLE IF NOT EXISTS web_cache ("
|
|
"key TEXT PRIMARY KEY, value TEXT NOT NULL, created REAL NOT NULL)"
|
|
)
|
|
conn.commit()
|
|
_schema_ready = True
|
|
return conn
|
|
|
|
|
|
def cache_key(prefix: str, payload: dict[str, Any]) -> str:
|
|
"""Deterministic cache key from a prefix and the normalized arguments."""
|
|
raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str)
|
|
digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32]
|
|
return f"{prefix}:{digest}"
|
|
|
|
|
|
def cache_get(key: str) -> Any | None:
|
|
"""Return the cached payload for *key*, or ``None`` (miss/expiry/disabled)."""
|
|
if ttl_seconds() <= 0:
|
|
return None
|
|
try:
|
|
conn = _connect()
|
|
row = conn.execute(
|
|
"SELECT value, created FROM web_cache WHERE key = ?", (key,)
|
|
).fetchone()
|
|
conn.close()
|
|
except sqlite3.Error as e:
|
|
logger.warning("web cache read failed (%s): %s", key, e)
|
|
return None
|
|
if row is None:
|
|
return None
|
|
value, created = row
|
|
if time.time() - float(created) > ttl_seconds():
|
|
return None
|
|
try:
|
|
return json.loads(value)
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def cache_set(key: str, value: Any) -> None:
|
|
"""Store *value* under *key* (best effort, never raises)."""
|
|
if ttl_seconds() <= 0:
|
|
return
|
|
try:
|
|
with _write_lock:
|
|
conn = _connect()
|
|
conn.execute(
|
|
"INSERT INTO web_cache (key, value, created) VALUES (?, ?, ?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value = excluded.value, created = excluded.created",
|
|
(key, json.dumps(value, ensure_ascii=False, default=str), time.time()),
|
|
)
|
|
conn.commit()
|
|
conn.close()
|
|
except sqlite3.Error as e:
|
|
logger.warning("web cache write failed (%s): %s", key, e)
|
|
|
|
|
|
def purge_expired() -> int:
|
|
"""Delete expired rows; return the number of removed entries (maintenance)."""
|
|
try:
|
|
conn = _connect()
|
|
cursor = conn.execute(
|
|
"DELETE FROM web_cache WHERE created < ?", (time.time() - ttl_seconds(),)
|
|
)
|
|
conn.commit()
|
|
deleted = cursor.rowcount
|
|
conn.close()
|
|
return int(deleted)
|
|
except sqlite3.Error as e:
|
|
logger.warning("web cache purge failed: %s", e)
|
|
return 0
|
|
|
|
|
|
def clear_cache() -> int:
|
|
"""Drop every cached entry (tests / admin); returns the number of rows."""
|
|
try:
|
|
conn = _connect()
|
|
cursor = conn.execute("DELETE FROM web_cache")
|
|
conn.commit()
|
|
deleted = cursor.rowcount
|
|
conn.close()
|
|
return int(deleted)
|
|
except sqlite3.Error as e:
|
|
logger.warning("web cache clear failed: %s", e)
|
|
return 0
|