feat(perf): chantier B — instantanés SQLite et rafraîchisseur d'arrière-plan
- Table snapshots + écriture traversante du cache client, hydratation au démarrage (SWR : servi + rafraîchi en fond). - Boucle de rafraîchissement (120 s par défaut) : index, étoiles, issues all+open et jalons des projets actifs ; webhook = rafraîchissement immédiat du dépôt ; fraîcheur affichée.
This commit is contained in:
+104
-13
@@ -64,6 +64,10 @@ class GiteaClient:
|
||||
self._cache: dict[str, tuple[float, Any]] = {}
|
||||
self._refreshing: set[str] = set()
|
||||
self._cache_gen = 0
|
||||
# Snapshot bookkeeping (chantier B): cache key -> UTC time
|
||||
# the payload was fetched from Gitea, for the freshness
|
||||
# label and the persisted `snapshots` table.
|
||||
self._snapshot_times: dict[str, str] = {}
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self.client.aclose()
|
||||
@@ -112,6 +116,90 @@ class GiteaClient:
|
||||
def _cache_put(self, key: str, value: Any) -> None:
|
||||
if self.cache_ttl > 0:
|
||||
self._cache[key] = (time.monotonic(), value)
|
||||
self._persist_snapshot(key, value)
|
||||
|
||||
# ── Snapshots (chantier B, 0.22.0) ──────────────────────
|
||||
|
||||
def _persist_snapshot(self, key: str, value: Any) -> None:
|
||||
"""Write-through to the SQLite snapshots table.
|
||||
|
||||
Persistence must never break a read: any failure (DB
|
||||
locked, unserializable payload) is logged and swallowed.
|
||||
"""
|
||||
try:
|
||||
from app import models
|
||||
|
||||
models.save_snapshot(key, value)
|
||||
import datetime
|
||||
|
||||
self._snapshot_times[key] = datetime.datetime.now(
|
||||
datetime.timezone.utc
|
||||
).strftime("%Y-%m-%d %H:%M:%S")
|
||||
except Exception as exc:
|
||||
logger.debug("snapshot persist failed for %s: %s", key, exc)
|
||||
|
||||
def hydrate_from_snapshots(self) -> int:
|
||||
"""Preload the cache from persisted snapshots (startup).
|
||||
|
||||
Entries are inserted with an already-stale timestamp: the
|
||||
SWR layer serves them instantly AND schedules a background
|
||||
refresh, so the first page after a restart renders from
|
||||
local data (tens of ms) while Gitea data catches up.
|
||||
Returns the number of entries hydrated.
|
||||
"""
|
||||
if self.cache_ttl <= 0:
|
||||
return 0
|
||||
try:
|
||||
from app import models
|
||||
|
||||
snapshots = models.load_snapshots()
|
||||
except Exception as exc:
|
||||
logger.warning("snapshot hydration failed: %s", exc)
|
||||
return 0
|
||||
stale_at = time.monotonic() - self.cache_ttl - 1.0
|
||||
for key, (payload, updated_at) in snapshots.items():
|
||||
self._cache[key] = (stale_at, payload)
|
||||
self._snapshot_times[key] = updated_at
|
||||
if snapshots:
|
||||
logger.info("hydrated %d snapshot(s) from SQLite", len(snapshots))
|
||||
return len(snapshots)
|
||||
|
||||
def latest_snapshot_at(self) -> str | None:
|
||||
"""Freshest Gitea fetch time across tracked keys (UTC str)."""
|
||||
if not self._snapshot_times:
|
||||
return None
|
||||
return max(self._snapshot_times.values())
|
||||
|
||||
async def _force(self, key: str, fetcher: Callable[[], Awaitable[Any]]) -> Any:
|
||||
"""Fetch now, bypassing freshness (background refresher)."""
|
||||
value = await fetcher()
|
||||
self._cache_put(key, value)
|
||||
return value
|
||||
|
||||
async def refresh_starred(self) -> list[dict]:
|
||||
return await self._force(
|
||||
"user:starred", lambda: self._get_all_pages("/user/starred")
|
||||
)
|
||||
|
||||
async def refresh_repo_issues(
|
||||
self, owner: str, repo: str, state: str = "all"
|
||||
) -> list[dict]:
|
||||
return await self._force(
|
||||
f"issues:{owner}/{repo}:{state}",
|
||||
lambda: self._get_all_pages(
|
||||
f"/repos/{owner}/{repo}/issues", state=state
|
||||
),
|
||||
)
|
||||
|
||||
async def refresh_milestones(
|
||||
self, owner: str, repo: str
|
||||
) -> list[dict]:
|
||||
return await self._force(
|
||||
f"milestones:{owner}/{repo}:open",
|
||||
lambda: self._get_all_pages(
|
||||
f"/repos/{owner}/{repo}/milestones", state="open"
|
||||
),
|
||||
)
|
||||
|
||||
def _cache_clear(self) -> None:
|
||||
self._cache.clear()
|
||||
@@ -182,22 +270,25 @@ class GiteaClient:
|
||||
self._user = await self._get("/user")
|
||||
return self._user
|
||||
|
||||
async def _fetch_search_repos(self) -> list[dict]:
|
||||
params: dict[str, Any] = {}
|
||||
try:
|
||||
user = await self.get_user()
|
||||
if user and user.get("id"):
|
||||
params["uid"] = user["id"]
|
||||
except Exception as exc: # no/invalid token → public search
|
||||
logger.debug(
|
||||
"get_user failed, falling back to public search: %s", exc
|
||||
)
|
||||
return await self._get_all_pages("/repos/search", **params)
|
||||
|
||||
async def search_repos(self) -> list[dict]:
|
||||
"""All repos visible to the token owner (paginated, SWR-cached)."""
|
||||
return await self._cached_list("search_repos", self._fetch_search_repos)
|
||||
|
||||
async def _fetch() -> list[dict]:
|
||||
params: dict[str, Any] = {}
|
||||
try:
|
||||
user = await self.get_user()
|
||||
if user and user.get("id"):
|
||||
params["uid"] = user["id"]
|
||||
except Exception as exc: # no/invalid token → public search
|
||||
logger.debug(
|
||||
"get_user failed, falling back to public search: %s", exc
|
||||
)
|
||||
return await self._get_all_pages("/repos/search", **params)
|
||||
|
||||
return await self._cached_list("search_repos", _fetch)
|
||||
async def refresh_search_repos(self) -> list[dict]:
|
||||
"""Force-fetch the repo index (background refresher)."""
|
||||
return await self._force("search_repos", self._fetch_search_repos)
|
||||
|
||||
async def get_repo(self, owner: str, repo: str) -> dict:
|
||||
return await self._get(f"/repos/{owner}/{repo}")
|
||||
|
||||
Reference in New Issue
Block a user