feat(youtube): InnerTube-first search, transcripts and watch-next related (Steps 15-18)

- InnerTube layer via pinned youtubei.js 18.1.0 (no quota, no key):
  search with merged continuations (unlimited pages), watch-next
  related with LockupView mapping, caption-track discovery
- 3-layer dispatcher (YT_SEARCH_MODE, default innertube-first):
  innertube -> yt-dlp scrape -> official API, graceful errors.yt
- Robust yt-dlp binary resolution (YT_DLP_PATH > PATH > bundled)
  with systematic API fallback (fixes spawn ENOENT in UI)
- Transcript: InnerTube caption discovery (YT_TRANSCRIPT_SOURCE),
  reusing pickTrack/orderedTracks/parseTrackText; yt-dlp fallback kept
- Watch: sidebar uses real watch-next related[] (/api/details),
  title-search fallback for other providers
- Cache: memory LRU + SQLite (youtube_search_cache, youtube_metrics),
  never persist empty pages; /healthz observability; /api/trending
- Includes pending Step 15/16 leftovers in same files (suggest,
  test scripts); unrelated provider adapters left uncommitted
This commit is contained in:
2026-09-25 19:38:46 -04:00
parent 300e5c45b1
commit 4bdcd393fe
25 changed files with 4112 additions and 168 deletions
+71
View File
@@ -998,3 +998,74 @@ function subscriptionRowToDto(row) {
channel,
};
}
// -------------------- Step 17 : cache persistant recherche YouTube --------------------
function ensureYoutubeCacheTables() {
try {
db.exec(`CREATE TABLE IF NOT EXISTS youtube_search_cache (
q_hash TEXT PRIMARY KEY, q TEXT NOT NULL, payload_json TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'scrape', created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL
);`);
db.exec(`CREATE INDEX IF NOT EXISTS idx_yt_cache_exp ON youtube_search_cache(expires_at);`);
db.exec(`CREATE TABLE IF NOT EXISTS youtube_metrics (
day TEXT PRIMARY KEY, scrape_calls INTEGER NOT NULL DEFAULT 0,
api_calls INTEGER NOT NULL DEFAULT 0, quota_units INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL
);`);
} catch {}
}
ensureYoutubeCacheTables();
export function getCachedYoutubeSearch(qHash) {
try {
ensureYoutubeCacheTables();
const row = db.prepare(`SELECT payload_json AS payload, source, expires_at AS exp FROM youtube_search_cache WHERE q_hash = ?`).get(qHash);
if (!row) return null;
if (Date.now() >= Number(row.exp || 0)) {
try { db.prepare(`DELETE FROM youtube_search_cache WHERE q_hash = ?`).run(qHash); } catch {}
return null;
}
try { return { items: JSON.parse(String(row.payload || '[]')), source: row.source }; } catch { return null; }
} catch { return null; }
}
export function setCachedYoutubeSearch(qHash, q, items, source, ttlMs) {
try {
ensureYoutubeCacheTables();
const now = Date.now();
db.prepare(`INSERT INTO youtube_search_cache (q_hash, q, payload_json, source, created_at, expires_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(q_hash) DO UPDATE SET q=excluded.q, payload_json=excluded.payload_json,
source=excluded.source, created_at=excluded.created_at, expires_at=excluded.expires_at`)
.run(qHash, String(q || '').slice(0, 300), JSON.stringify(items || []), source, now, now + ttlMs);
// Cap : garde 2000 entrées les plus fraîches
try { db.exec(`DELETE FROM youtube_search_cache WHERE q_hash NOT IN (SELECT q_hash FROM youtube_search_cache ORDER BY expires_at DESC LIMIT 2000)`); } catch {}
} catch {}
}
export function pruneYoutubeCache() {
try { db.prepare(`DELETE FROM youtube_search_cache WHERE expires_at <= ?`).run(Date.now()); } catch {}
}
export function incYoutubeMetrics({ scrapeCalls = 0, apiCalls = 0, quotaUnits = 0 } = {}) {
try {
ensureYoutubeCacheTables();
const day = new Date().toISOString().slice(0, 10);
db.prepare(`INSERT INTO youtube_metrics (day, scrape_calls, api_calls, quota_units, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(day) DO UPDATE SET scrape_calls = scrape_calls + ?, api_calls = api_calls + ?,
quota_units = quota_units + ?, updated_at = excluded.updated_at`)
.run(day, scrapeCalls, apiCalls, quotaUnits, new Date().toISOString(), scrapeCalls, apiCalls, quotaUnits);
} catch {}
}
export function getYoutubeMetricsToday() {
try {
ensureYoutubeCacheTables();
const day = new Date().toISOString().slice(0, 10);
return db.prepare(`SELECT * FROM youtube_metrics WHERE day = ?`).get(day) || { day, scrape_calls: 0, api_calls: 0, quota_units: 0 };
} catch { return { scrape_calls: 0, api_calls: 0, quota_units: 0 }; }
}
export function countYoutubeCacheRows() {
try { return db.prepare(`SELECT COUNT(1) AS n FROM youtube_search_cache`).get()?.n || 0; } catch { return 0; }
}