feat(youtube): InnerTube-first search, transcripts and watch-next related (Steps 15-18)
- InnerTube layer via pinned youtubei.js 18.1.0 (no quota, no key): search with merged continuations (unlimited pages), watch-next related with LockupView mapping, caption-track discovery - 3-layer dispatcher (YT_SEARCH_MODE, default innertube-first): innertube -> yt-dlp scrape -> official API, graceful errors.yt - Robust yt-dlp binary resolution (YT_DLP_PATH > PATH > bundled) with systematic API fallback (fixes spawn ENOENT in UI) - Transcript: InnerTube caption discovery (YT_TRANSCRIPT_SOURCE), reusing pickTrack/orderedTracks/parseTrackText; yt-dlp fallback kept - Watch: sidebar uses real watch-next related[] (/api/details), title-search fallback for other providers - Cache: memory LRU + SQLite (youtube_search_cache, youtube_metrics), never persist empty pages; /healthz observability; /api/trending - Includes pending Step 15/16 leftovers in same files (suggest, test scripts); unrelated provider adapters left uncommitted
This commit is contained in:
@@ -998,3 +998,74 @@ function subscriptionRowToDto(row) {
|
||||
channel,
|
||||
};
|
||||
}
|
||||
|
||||
// -------------------- Step 17 : cache persistant recherche YouTube --------------------
|
||||
function ensureYoutubeCacheTables() {
|
||||
try {
|
||||
db.exec(`CREATE TABLE IF NOT EXISTS youtube_search_cache (
|
||||
q_hash TEXT PRIMARY KEY, q TEXT NOT NULL, payload_json TEXT NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'scrape', created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL
|
||||
);`);
|
||||
db.exec(`CREATE INDEX IF NOT EXISTS idx_yt_cache_exp ON youtube_search_cache(expires_at);`);
|
||||
db.exec(`CREATE TABLE IF NOT EXISTS youtube_metrics (
|
||||
day TEXT PRIMARY KEY, scrape_calls INTEGER NOT NULL DEFAULT 0,
|
||||
api_calls INTEGER NOT NULL DEFAULT 0, quota_units INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL
|
||||
);`);
|
||||
} catch {}
|
||||
}
|
||||
ensureYoutubeCacheTables();
|
||||
|
||||
export function getCachedYoutubeSearch(qHash) {
|
||||
try {
|
||||
ensureYoutubeCacheTables();
|
||||
const row = db.prepare(`SELECT payload_json AS payload, source, expires_at AS exp FROM youtube_search_cache WHERE q_hash = ?`).get(qHash);
|
||||
if (!row) return null;
|
||||
if (Date.now() >= Number(row.exp || 0)) {
|
||||
try { db.prepare(`DELETE FROM youtube_search_cache WHERE q_hash = ?`).run(qHash); } catch {}
|
||||
return null;
|
||||
}
|
||||
try { return { items: JSON.parse(String(row.payload || '[]')), source: row.source }; } catch { return null; }
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
export function setCachedYoutubeSearch(qHash, q, items, source, ttlMs) {
|
||||
try {
|
||||
ensureYoutubeCacheTables();
|
||||
const now = Date.now();
|
||||
db.prepare(`INSERT INTO youtube_search_cache (q_hash, q, payload_json, source, created_at, expires_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(q_hash) DO UPDATE SET q=excluded.q, payload_json=excluded.payload_json,
|
||||
source=excluded.source, created_at=excluded.created_at, expires_at=excluded.expires_at`)
|
||||
.run(qHash, String(q || '').slice(0, 300), JSON.stringify(items || []), source, now, now + ttlMs);
|
||||
// Cap : garde 2000 entrées les plus fraîches
|
||||
try { db.exec(`DELETE FROM youtube_search_cache WHERE q_hash NOT IN (SELECT q_hash FROM youtube_search_cache ORDER BY expires_at DESC LIMIT 2000)`); } catch {}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
export function pruneYoutubeCache() {
|
||||
try { db.prepare(`DELETE FROM youtube_search_cache WHERE expires_at <= ?`).run(Date.now()); } catch {}
|
||||
}
|
||||
|
||||
export function incYoutubeMetrics({ scrapeCalls = 0, apiCalls = 0, quotaUnits = 0 } = {}) {
|
||||
try {
|
||||
ensureYoutubeCacheTables();
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
db.prepare(`INSERT INTO youtube_metrics (day, scrape_calls, api_calls, quota_units, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(day) DO UPDATE SET scrape_calls = scrape_calls + ?, api_calls = api_calls + ?,
|
||||
quota_units = quota_units + ?, updated_at = excluded.updated_at`)
|
||||
.run(day, scrapeCalls, apiCalls, quotaUnits, new Date().toISOString(), scrapeCalls, apiCalls, quotaUnits);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
export function getYoutubeMetricsToday() {
|
||||
try {
|
||||
ensureYoutubeCacheTables();
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
return db.prepare(`SELECT * FROM youtube_metrics WHERE day = ?`).get(day) || { day, scrape_calls: 0, api_calls: 0, quota_units: 0 };
|
||||
} catch { return { scrape_calls: 0, api_calls: 0, quota_units: 0 }; }
|
||||
}
|
||||
|
||||
export function countYoutubeCacheRows() {
|
||||
try { return db.prepare(`SELECT COUNT(1) AS n FROM youtube_search_cache`).get()?.n || 0; } catch { return 0; }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user