feat(youtube): InnerTube-first search, transcripts and watch-next related (Steps 15-18)
- InnerTube layer via pinned youtubei.js 18.1.0 (no quota, no key): search with merged continuations (unlimited pages), watch-next related with LockupView mapping, caption-track discovery - 3-layer dispatcher (YT_SEARCH_MODE, default innertube-first): innertube -> yt-dlp scrape -> official API, graceful errors.yt - Robust yt-dlp binary resolution (YT_DLP_PATH > PATH > bundled) with systematic API fallback (fixes spawn ENOENT in UI) - Transcript: InnerTube caption discovery (YT_TRANSCRIPT_SOURCE), reusing pickTrack/orderedTracks/parseTrackText; yt-dlp fallback kept - Watch: sidebar uses real watch-next related[] (/api/details), title-search fallback for other providers - Cache: memory LRU + SQLite (youtube_search_cache, youtube_metrics), never persist empty pages; /healthz observability; /api/trending - Includes pending Step 15/16 leftovers in same files (suggest, test scripts); unrelated provider adapters left uncommitted
This commit is contained in:
+458
-3
@@ -7,12 +7,16 @@ import bcrypt from 'bcryptjs';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import youtubedl from 'youtube-dl-exec';
|
||||
import youtubedlPkg, { create as createYtDlp } from 'youtube-dl-exec';
|
||||
import { execFile as execFileCb } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { fileURLToPath as serverFileURLToPath } from 'node:url';
|
||||
import ffmpegPath from 'ffmpeg-static';
|
||||
import * as cheerio from 'cheerio';
|
||||
import axios from 'axios';
|
||||
import rumbleRouter from './rumble.mjs';
|
||||
import { providerRegistry, validateProviders } from './providers/registry.mjs';
|
||||
import { pickTrack, parseTrackText, parseVtt, orderedTracks, normalizeTranscriptProvider, transcriptTrackExt, looksLikeHtmlError } from './transcript.mjs';
|
||||
import {
|
||||
getUserByUsername,
|
||||
getUserById,
|
||||
@@ -73,9 +77,53 @@ import {
|
||||
} from './db.mjs';
|
||||
import { getChannelAdapter, setTwitchTokenProvider } from './providers/channel-registry.mjs';
|
||||
import { fetchChannelContent } from './providers/channel-content.mjs';
|
||||
import { getSearchMode, getYtDlpBin, hasCookiesFile, metricsSnapshot } from './providers/youtube-common.mjs';
|
||||
import { ytScrapeCacheStats } from './providers/youtube.mjs';
|
||||
|
||||
const app = express();
|
||||
const PORT = Number(process.env.PORT || 4000);
|
||||
// yt-dlp: prefer the newest binary available. The copy bundled with
|
||||
// youtube-dl-exec goes stale (YouTube then answers "The page needs to be
|
||||
// reloaded" to dump-single-json), while a system install is usually fresher.
|
||||
// `YT_DLP_PATH` wins, then `yt-dlp` on PATH, then the bundled binary.
|
||||
const execFileAsync = promisify(execFileCb);
|
||||
async function binaryVersion(bin) {
|
||||
try {
|
||||
const { stdout } = await execFileAsync(bin, ['--version'], { timeout: 15000 });
|
||||
return String(stdout || '').trim().split('\n')[0].trim();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
let youtubedl = youtubedlPkg;
|
||||
let ytDlpInfo = 'bundled';
|
||||
try {
|
||||
let bundledBin = null;
|
||||
try {
|
||||
const u = new URL('../node_modules/youtube-dl-exec/bin/', import.meta.url);
|
||||
const cand = path.join(String(serverFileURLToPath(u)), process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp');
|
||||
if (fs.existsSync(cand)) bundledBin = cand;
|
||||
} catch {}
|
||||
const bundledVer = bundledBin ? await binaryVersion(bundledBin) : null;
|
||||
const candidates = [];
|
||||
if (process.env.YT_DLP_PATH) candidates.push(process.env.YT_DLP_PATH);
|
||||
candidates.push(process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp', 'yt-dlp');
|
||||
let best = null;
|
||||
for (const cand of candidates) {
|
||||
if (!cand) continue;
|
||||
const ver = await binaryVersion(cand);
|
||||
if (ver && (!best || ver > best.ver)) best = { bin: cand, ver };
|
||||
}
|
||||
if (best && (!bundledVer || best.ver >= bundledVer)) {
|
||||
youtubedl = createYtDlp(best.bin);
|
||||
ytDlpInfo = `${best.bin} (${best.ver})`;
|
||||
} else if (bundledVer) {
|
||||
ytDlpInfo = `bundled (${bundledVer})`;
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[config] yt-dlp binary selection failed, using bundled:', e?.message || e);
|
||||
}
|
||||
console.log(`[config] yt-dlp: ${ytDlpInfo}`);
|
||||
const IS_PROD = String(process.env.NODE_ENV || '').toLowerCase() === 'production';
|
||||
const JWT_SECRET = process.env.JWT_SECRET || 'dev-secret-change-me';
|
||||
if (!process.env.JWT_SECRET) {
|
||||
@@ -1006,6 +1054,14 @@ r.get('/peertube/:instance/*', async (req, res) => {
|
||||
});
|
||||
|
||||
// -------------------- Generic video details (GET) --------------------
|
||||
// Cache mémoire pour les vidéos connexes InnerTube (watch-next) : TTL 1h, LRU 200.
|
||||
const YT_RELATED_TTL_MS = Number(process.env.YT_RELATED_TTL_MS || 60 * 60 * 1000);
|
||||
const ytRelatedCache = new Map();
|
||||
function ytRelatedCacheSet(key, items) {
|
||||
if (ytRelatedCache.has(key)) ytRelatedCache.delete(key);
|
||||
ytRelatedCache.set(key, { ts: Date.now(), items });
|
||||
while (ytRelatedCache.size > 200) { const o = ytRelatedCache.keys().next().value; if (o === undefined) break; ytRelatedCache.delete(o); }
|
||||
}
|
||||
// Returns metadata such as title, description, uploader, thumbnail, duration and views for a provider/videoId
|
||||
// Supports query params similar to download endpoints: instance (PeerTube), slug (Odysee), sourceUrl (direct)
|
||||
r.get('/details/:provider/:videoId', async (req, res) => {
|
||||
@@ -1071,6 +1127,22 @@ r.get('/details/:provider/:videoId', async (req, res) => {
|
||||
url,
|
||||
type: 'video',
|
||||
};
|
||||
// Step 18 : vidéos connexes façon SmartTube (watch-next InnerTube, best-effort, 0 quota).
|
||||
// N'implique que YouTube ; toute erreur -> `related: []`, la réponse reste 200.
|
||||
if (String(provider) === 'youtube' && req.query.related !== '0') {
|
||||
try {
|
||||
const { getRelatedViaInnerTube } = await import('./providers/youtube-innertube.mjs');
|
||||
const relKey = `related:${videoId}`;
|
||||
const cached = ytRelatedCache.get(relKey);
|
||||
if (cached && (Date.now() - cached.ts) < YT_RELATED_TTL_MS) {
|
||||
out.related = cached.items;
|
||||
} else {
|
||||
const items = await getRelatedViaInnerTube(videoId, 24).catch(() => []);
|
||||
out.related = items;
|
||||
ytRelatedCacheSet(relKey, items);
|
||||
}
|
||||
} catch { out.related = []; }
|
||||
}
|
||||
return res.json(out);
|
||||
} catch (e) {
|
||||
return res.status(500).json({ error: 'details_failed', details: String(e?.message || e) });
|
||||
@@ -1684,7 +1756,7 @@ r.post('/telemetry/events', authMiddleware, telemetryLimiter, (req, res) => {
|
||||
const { event, meta } = req.body || {};
|
||||
if (!event || typeof event !== 'string') return res.status(400).json({ error: 'event_required' });
|
||||
// Whitelist known event names to keep the table clean
|
||||
const allowed = new Set(['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open']);
|
||||
const allowed = new Set(['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open', 'suggest_shown', 'suggest_used']);
|
||||
if (!allowed.has(event)) return res.status(400).json({ error: 'unknown_event' });
|
||||
const row = insertTelemetryEvent({ userId: req.user.id, event, meta: (meta && typeof meta === 'object') ? meta : null });
|
||||
return res.status(201).json(row || { ok: true });
|
||||
@@ -1704,7 +1776,7 @@ r.get('/telemetry/events', authMiddleware, (req, res) => {
|
||||
r.get('/telemetry/summary', authMiddleware, (req, res) => {
|
||||
const since = typeof req.query.since === 'string' ? req.query.since : undefined;
|
||||
const events = {};
|
||||
for (const name of ['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open']) {
|
||||
for (const name of ['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open', 'suggest_shown', 'suggest_used']) {
|
||||
events[name] = countTelemetryEvents({ event: name, since });
|
||||
}
|
||||
return res.json({ events });
|
||||
@@ -1944,6 +2016,53 @@ r.get('/img/odysee', async (req, res) => {
|
||||
app.use('/api', r);
|
||||
// Health endpoint for container checks
|
||||
app.get('/api/health', (_req, res) => res.json({ status: 'ok' }));
|
||||
// Step 17 : observabilité YouTube (mode, yt-dlp, cache, quota). Aucun secret exposé.
|
||||
app.get(['/healthz', '/api/healthz'], async (_req, res) => {
|
||||
try {
|
||||
const [{ getYoutubeMetricsToday, countYoutubeCacheRows }, common] =
|
||||
await Promise.all([import('./db.mjs'), import('./providers/youtube-common.mjs')]);
|
||||
let ytdlpVersion = null;
|
||||
let resolvedBin = null;
|
||||
try {
|
||||
resolvedBin = await common.resolveYtDlpBin();
|
||||
const { stdout } = await execFileAsync(resolvedBin, ['--version'], { timeout: 10000 });
|
||||
ytdlpVersion = String(stdout || '').trim().split('\n')[0].trim() || null;
|
||||
} catch {}
|
||||
const keys = common.getYouTubeKeys();
|
||||
const banned = [];
|
||||
try { for (const [k, until] of ytKeyBans || []) if (Date.now() < until) banned.push(`...${String(k).slice(-4)}`); } catch {}
|
||||
res.json({
|
||||
status: 'ok',
|
||||
youtube: {
|
||||
mode: getSearchMode(),
|
||||
ytdlp: { bin: resolvedBin || getYtDlpBin(), version: ytdlpVersion, info: ytDlpInfo, binOk: Boolean(ytdlpVersion) },
|
||||
antiban: {
|
||||
cookiesFile: hasCookiesFile(),
|
||||
poToken: Boolean(String(process.env.YT_PO_TOKEN || '').trim()),
|
||||
egressProxy: Boolean(String(process.env.YT_EGRESS_PROXY || '').trim()),
|
||||
},
|
||||
cache: { ...ytScrapeCacheStats(), sqliteRows: countYoutubeCacheRows() },
|
||||
metrics: { ...metricsSnapshot(), today: getYoutubeMetricsToday() },
|
||||
keys: { count: keys.length, banned },
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
res.status(500).json({ status: 'error', error: String(e?.message || e) });
|
||||
}
|
||||
});
|
||||
// Step 17 : trending YouTube sans clé (scrape) avec fallback [] propre.
|
||||
app.get('/api/trending', async (req, res) => {
|
||||
try {
|
||||
const provider = String(req.query.provider || 'yt');
|
||||
const limit = Math.min(50, Math.max(1, Number(req.query.limit || 24)));
|
||||
if (provider !== 'yt') return res.status(400).json({ error: 'only yt supported in phase 1' });
|
||||
const { getTrendingViaScrape } = await import('./providers/youtube-scrape.mjs');
|
||||
const items = await getTrendingViaScrape(limit);
|
||||
return res.json({ provider, items });
|
||||
} catch (e) {
|
||||
return res.json({ provider: 'yt', items: [], error: String(e?.code || e?.message || 'trending_failed') });
|
||||
}
|
||||
});
|
||||
// Alias to support Angular dev proxy paths in both dev and production builds
|
||||
app.use('/proxy/api', r);
|
||||
// Mount dedicated Rumble router (browse, search, video)
|
||||
@@ -2170,6 +2289,342 @@ app.get('/api/search', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// -------------------- Query typeahead suggestions (Step 15) --------------------
|
||||
// GET /api/search/suggest?q=…&providers=yt,dm&limit=10 -> { q, groups: { yt: string[], dm: string[] } }
|
||||
// Fan-out over provider `suggest()` handlers; providers without one degrade to [] (never 500).
|
||||
const SUGGEST_CACHE_TTL_MS = Number(process.env.SUGGEST_CACHE_TTL_MS || 5 * 60 * 1000);
|
||||
const SUGGEST_CACHE_MAX_ENTRIES = 500;
|
||||
/** @type {Map<string, { ts: number, data: any }>} */
|
||||
const suggestCache = new Map();
|
||||
function suggestCacheGet(key) {
|
||||
const hit = suggestCache.get(key);
|
||||
if (!hit) return null;
|
||||
if ((Date.now() - hit.ts) >= SUGGEST_CACHE_TTL_MS) {
|
||||
suggestCache.delete(key);
|
||||
return null;
|
||||
}
|
||||
// LRU refresh
|
||||
suggestCache.delete(key);
|
||||
suggestCache.set(key, hit);
|
||||
return hit.data;
|
||||
}
|
||||
function suggestCacheSet(key, data) {
|
||||
if (suggestCache.has(key)) suggestCache.delete(key);
|
||||
suggestCache.set(key, { ts: Date.now(), data });
|
||||
while (suggestCache.size > SUGGEST_CACHE_MAX_ENTRIES) {
|
||||
const oldest = suggestCache.keys().next().value;
|
||||
suggestCache.delete(oldest);
|
||||
}
|
||||
}
|
||||
const suggestLimiter = rateLimit({
|
||||
windowMs: 60 * 1000,
|
||||
max: Number(process.env.SUGGEST_RATE_LIMIT || 60),
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
});
|
||||
app.get('/api/search/suggest', suggestLimiter, async (req, res) => {
|
||||
try {
|
||||
const rawQ = typeof req.query.q === 'string' ? req.query.q : '';
|
||||
const q = rawQ.trim();
|
||||
if (q.length < 2) {
|
||||
return res.status(400).json({ error: 'q is required and must be at least 2 characters long' });
|
||||
}
|
||||
const limit = Math.min(20, Math.max(1, Number(req.query.limit || 10)));
|
||||
const requested = typeof req.query.providers === 'string' ? String(req.query.providers) : '';
|
||||
const validProviders = validateProviders(requested);
|
||||
const cacheKey = `suggest:${validProviders.join(',')}:${q.toLowerCase()}:${limit}`;
|
||||
const cached = suggestCacheGet(cacheKey);
|
||||
if (cached) return res.json(cached);
|
||||
const results = await Promise.allSettled(
|
||||
validProviders.map((providerId) => {
|
||||
const mod = providerRegistry[providerId];
|
||||
if (!mod || typeof mod.suggest !== 'function') return Promise.resolve([]);
|
||||
return Promise.resolve().then(() => mod.suggest(q, { limit }));
|
||||
})
|
||||
);
|
||||
const groups = {};
|
||||
results.forEach((result, index) => {
|
||||
const providerId = validProviders[index];
|
||||
if (result.status === 'fulfilled' && Array.isArray(result.value)) {
|
||||
groups[providerId] = result.value
|
||||
.map((s) => String(s ?? '').trim())
|
||||
.filter(Boolean)
|
||||
.slice(0, limit);
|
||||
} else {
|
||||
groups[providerId] = [];
|
||||
}
|
||||
});
|
||||
const data = { q, groups };
|
||||
suggestCacheSet(cacheKey, data);
|
||||
return res.json(data);
|
||||
} catch (e) {
|
||||
return res.status(500).json({ error: 'suggest_failed', details: String(e?.message || e) });
|
||||
}
|
||||
});
|
||||
|
||||
// -------------------- Video transcripts (Step 16, Phase 1) --------------------
|
||||
// GET /api/transcript/:provider/:videoId?lang=&instance=&slug=&sourceUrl=
|
||||
// -> { lang, available, languages, lines: [{ t, dur, text }] }
|
||||
// One endpoint / one parser / one UI whatever the provider (yt-dlp subtitles).
|
||||
const TRANSCRIPT_CACHE_TTL_MS = Number(process.env.TRANSCRIPT_CACHE_TTL || 24 * 60 * 60 * 1000);
|
||||
const TRANSCRIPT_CACHE_MAX_ENTRIES = 200;
|
||||
/** @type {Map<string, { ts: number, data: any }>} */
|
||||
const transcriptCache = new Map();
|
||||
function transcriptCacheGet(key) {
|
||||
const hit = transcriptCache.get(key);
|
||||
if (!hit) return null;
|
||||
if ((Date.now() - hit.ts) >= TRANSCRIPT_CACHE_TTL_MS) {
|
||||
transcriptCache.delete(key);
|
||||
return null;
|
||||
}
|
||||
transcriptCache.delete(key);
|
||||
transcriptCache.set(key, hit);
|
||||
return hit.data;
|
||||
}
|
||||
function transcriptCacheSet(key, data) {
|
||||
if (transcriptCache.has(key)) transcriptCache.delete(key);
|
||||
transcriptCache.set(key, { ts: Date.now(), data });
|
||||
while (transcriptCache.size > TRANSCRIPT_CACHE_MAX_ENTRIES) {
|
||||
const oldest = transcriptCache.keys().next().value;
|
||||
transcriptCache.delete(oldest);
|
||||
}
|
||||
}
|
||||
const transcriptLimiter = rateLimit({
|
||||
windowMs: 60 * 1000,
|
||||
max: Number(process.env.TRANSCRIPT_RATE_LIMIT || 10),
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
// Always answer JSON (default handler sends an HTML/text page, which the
|
||||
// Angular HttpClient cannot parse as JSON and surfaces as a raw SyntaxError).
|
||||
handler: (req, res) => {
|
||||
return res.status(429).json({ available: false, error: 'rate_limited' });
|
||||
},
|
||||
});
|
||||
|
||||
/** Providers known NOT to expose subtitle tracks via yt-dlp (no transcript possible). */
|
||||
const TRANSCRIPT_UNSUPPORTED_PROVIDERS = new Set(['twitch', 'odysee', 'rumble']);
|
||||
|
||||
/** Download subtitles via yt-dlp (handles YouTube impersonation + 429-prone
|
||||
* translated tracks). Tries `langs` in order, returns the first non-empty
|
||||
* parsed lines with the language that worked, or null. Bounded by a timeout
|
||||
* (yt-dlp subtitle downloads can hang on .part files); partial results on
|
||||
* disk are still scanned when yt-dlp exits non-zero. */
|
||||
async function transcriptViaYtDlp(url, langs) {
|
||||
const osMod = await import('node:os');
|
||||
const dir = await fs.promises.mkdtemp(path.join(osMod.tmpdir(), 'newtube-transcript-'));
|
||||
const scanDir = async (wanted) => {
|
||||
let files = [];
|
||||
try {
|
||||
files = (await fs.promises.readdir(dir)).filter((f) => /\.vtt$/i.test(f) && !/\.part$/i.test(f));
|
||||
} catch { return null; }
|
||||
// Ignore stale/empty files
|
||||
const nonEmpty = [];
|
||||
for (const f of files) {
|
||||
try {
|
||||
const st = await fs.promises.stat(path.join(dir, f));
|
||||
if (st.size > 0) nonEmpty.push(f);
|
||||
} catch {}
|
||||
}
|
||||
// Prefer requested languages first
|
||||
nonEmpty.sort((a, b) => {
|
||||
const la = a.toLowerCase(), lb = b.toLowerCase();
|
||||
const ia = wanted.findIndex((w) => la.includes(`.${w.toLowerCase()}.`) || la.endsWith(`.${w.toLowerCase()}.vtt`));
|
||||
const ib = wanted.findIndex((w) => lb.includes(`.${w.toLowerCase()}.`) || lb.endsWith(`.${w.toLowerCase()}.vtt`));
|
||||
return (ia === -1 ? 99 : ia) - (ib === -1 ? 99 : ib);
|
||||
});
|
||||
for (const file of nonEmpty) {
|
||||
try {
|
||||
const text = await fs.promises.readFile(path.join(dir, file), 'utf8');
|
||||
const lines = parseVtt(text);
|
||||
if (lines.length) {
|
||||
const m = /\.([a-z]{2,3}(?:-[a-z]{2,4})?)\.vtt$/i.exec(file);
|
||||
return { lines, lang: m ? m[1] : null };
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
try {
|
||||
const wanted = Array.from(new Set((langs || []).map((l) => String(l || '').split('-')[0]).filter(Boolean)));
|
||||
if (!wanted.includes('en')) wanted.push('en');
|
||||
const subLangs = wanted.slice(0, 4).join(',');
|
||||
const child = youtubedl(url, {
|
||||
writeSub: true,
|
||||
writeAutoSub: true,
|
||||
subLangs,
|
||||
subFormat: 'vtt/best',
|
||||
skipDownload: true,
|
||||
noWarnings: true,
|
||||
noCheckCertificates: true,
|
||||
noPlaylist: true,
|
||||
output: path.join(dir, '%(id)s'),
|
||||
});
|
||||
const timer = setTimeout(() => { try { child.kill('SIGKILL'); } catch {} }, 90000);
|
||||
try {
|
||||
await child;
|
||||
} catch (e) {
|
||||
// Non-zero exit (e.g. one language 429'd) — partial files may still exist.
|
||||
console.warn('[transcript] yt-dlp subtitle download exited non-zero:', String(e?.message || e).slice(0, 200));
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
return await scanDir(wanted);
|
||||
} catch (e) {
|
||||
console.warn('[transcript] yt-dlp subtitle download failed:', e?.message || e);
|
||||
try {
|
||||
const wanted = Array.from(new Set((langs || []).map((l) => String(l || '').split('-')[0]).filter(Boolean)));
|
||||
return await scanDir(wanted);
|
||||
} catch { return null; }
|
||||
} finally {
|
||||
try { await fs.promises.rm(dir, { recursive: true, force: true }); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
/** Fetch one timedtext track URL and parse it (any format). Returns [] on any failure. */
|
||||
async function fetchTimedTextLines(track) {
|
||||
const resp = await fetch(String(track.url), {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
|
||||
Accept: 'application/json, text/vtt, text/*;q=0.9, */*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9,fr;q=0.8',
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`track_fetch_failed:${resp.status}`);
|
||||
const text = await resp.text();
|
||||
if (!text || looksLikeHtmlError(text)) throw new Error('track_fetch_failed:html_error_page');
|
||||
return parseTrackText(text, transcriptTrackExt(track));
|
||||
}
|
||||
|
||||
app.get('/api/transcript/:provider/:videoId', transcriptLimiter, async (req, res) => {
|
||||
try {
|
||||
const { provider, videoId } = req.params;
|
||||
const lang = String(req.query.lang || 'fr').slice(0, 12) || 'fr';
|
||||
const normalized = normalizeTranscriptProvider(provider);
|
||||
if (!normalized || !videoId) {
|
||||
return res.status(400).json({ available: false, error: 'invalid_provider_or_video' });
|
||||
}
|
||||
const cacheKey = `transcript:${normalized}:${videoId}:${lang.toLowerCase()}`;
|
||||
const cached = transcriptCacheGet(cacheKey);
|
||||
if (cached) return res.json(cached);
|
||||
// Source de découverte des pistes : InnerTube d'abord pour YouTube
|
||||
// (mêmes URLs timedtext, sans spawn yt-dlp), yt-dlp sinon/en secours.
|
||||
// YT_TRANSCRIPT_SOURCE=innertube-first (défaut) | ytdlp-only | innertube-only
|
||||
const transcriptSource = String(process.env.YT_TRANSCRIPT_SOURCE || 'innertube-first').trim().toLowerCase();
|
||||
const transcriptUrl = providerUrlFrom(normalized, String(videoId), {
|
||||
instance: req.query.instance || undefined,
|
||||
slug: req.query.slug || undefined,
|
||||
sourceUrl: req.query.sourceUrl || undefined,
|
||||
});
|
||||
let meta = null;
|
||||
if (normalized === 'youtube' && transcriptSource !== 'ytdlp-only') {
|
||||
try {
|
||||
const { getCaptionTracksViaInnerTube } = await import('./providers/youtube-innertube.mjs');
|
||||
const cap = await getCaptionTracksViaInnerTube(String(videoId));
|
||||
if (cap.trackCount > 0) {
|
||||
meta = { subtitles: cap.subtitles, automatic_captions: cap.automatic_captions };
|
||||
console.log(`[transcript] source=innertube tracks=${cap.trackCount} langs=${cap.languages.join(',')}`);
|
||||
} else {
|
||||
// InnerTube fait foi (même backend que le lecteur) : 0 piste = pas
|
||||
// de sous-titres, sans payer un dump yt-dlp complet.
|
||||
console.log('[transcript] source=innertube tracks=0 -> no_subtitles');
|
||||
const empty = { lang: null, available: false, languages: [], lines: [], reason: 'no_subtitles' };
|
||||
transcriptCacheSet(cacheKey, empty);
|
||||
return res.json(empty);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[transcript] innertube discovery failed, fallback yt-dlp:', e?.code || String(e?.message || e).slice(0, 120));
|
||||
if (transcriptSource === 'innertube-only') {
|
||||
return res.status(502).json({ available: false, languages: [], lines: [], error: 'transcript_temporarily_unavailable', retryable: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!meta) {
|
||||
try {
|
||||
const raw = await youtubedl(transcriptUrl, { dumpSingleJson: true, skipDownload: true, noWarnings: true, noCheckCertificates: true });
|
||||
meta = (typeof raw === 'string') ? JSON.parse(raw || '{}') : (raw || {});
|
||||
} catch (e) {
|
||||
console.error('[transcript] yt-dlp failed:', e?.message || e);
|
||||
return res.status(502).json({ available: false, languages: [], lines: [], error: 'transcript_temporarily_unavailable', retryable: true });
|
||||
}
|
||||
}
|
||||
const { track, languages, lang: chosenLang } = pickTrack(meta, lang);
|
||||
if (!track) {
|
||||
// Definitive absence: distinguish "provider never exposes subtitles"
|
||||
// (twitch/odysee/rumble) from "this video has none" — cacheable 200s.
|
||||
const reason = TRANSCRIPT_UNSUPPORTED_PROVIDERS.has(normalized) ? 'provider_unsupported' : 'no_subtitles';
|
||||
const empty = { lang: null, available: false, languages: languages || [], lines: [], reason };
|
||||
transcriptCacheSet(cacheKey, empty);
|
||||
return res.json(empty);
|
||||
}
|
||||
// Try candidate tracks in order: requested language, then original (`en`),
|
||||
// then everything else. Translated tracks are often rate-limited while the
|
||||
// original still works — never fail on the first track alone.
|
||||
// Bounded: hammering dozens of timedtext URLs only worsens YouTube 429s.
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
let lines = [];
|
||||
let workedLang = chosenLang;
|
||||
let sawRateLimit = false;
|
||||
for (const cand of orderedTracks(meta, lang).slice(0, 5)) {
|
||||
let attempt = 0;
|
||||
for (;;) {
|
||||
try {
|
||||
const parsed = await fetchTimedTextLines(cand.track);
|
||||
if (parsed && parsed.length) {
|
||||
lines = parsed;
|
||||
workedLang = cand.lang || chosenLang;
|
||||
}
|
||||
break;
|
||||
} catch (e) {
|
||||
const msg = String(e?.message || e);
|
||||
console.warn('[transcript] track fetch failed:', cand.lang, msg);
|
||||
// One retry after a short pause on transient 429s.
|
||||
if (msg.includes(':429') && attempt === 0) {
|
||||
sawRateLimit = true;
|
||||
attempt += 1;
|
||||
await sleep(1500);
|
||||
continue;
|
||||
}
|
||||
if (msg.includes(':429')) sawRateLimit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lines.length) break;
|
||||
}
|
||||
if (!lines || lines.length === 0) {
|
||||
// Direct timedtext fetches failed (429 / Sorry pages / impersonation):
|
||||
// let yt-dlp download the subtitles instead (handles all of the above).
|
||||
try {
|
||||
const viaDlp = await transcriptViaYtDlp(transcriptUrl, [lang, chosenLang].filter(Boolean));
|
||||
if (viaDlp && viaDlp.lines && viaDlp.lines.length) {
|
||||
lines = viaDlp.lines;
|
||||
if (viaDlp.lang) workedLang = viaDlp.lang;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
if (!lines || lines.length === 0) {
|
||||
// Subtitle tracks exist but their content could not be retrieved or
|
||||
// parsed (YouTube 429 / Sorry pages / impersonation): the video HAS
|
||||
// subtitles, so this is transient — 502 with languages + retryable flag,
|
||||
// never cached as "no subtitles".
|
||||
return res.status(502).json({
|
||||
available: false,
|
||||
languages: languages || [],
|
||||
lines: [],
|
||||
error: 'transcript_temporarily_unavailable',
|
||||
retryable: true,
|
||||
rateLimited: sawRateLimit || undefined,
|
||||
});
|
||||
}
|
||||
const result = { lang: workedLang, available: true, languages: languages || [], lines };
|
||||
transcriptCacheSet(cacheKey, result);
|
||||
return res.json(result);
|
||||
} catch (e) {
|
||||
console.error('[transcript] unexpected error:', e?.message || e);
|
||||
return res.status(502).json({ available: false, languages: [], lines: [], error: 'transcript_temporarily_unavailable', retryable: true });
|
||||
}
|
||||
});
|
||||
|
||||
// -------------------- Static Frontend (Angular build) --------------------
|
||||
const distRoot = path.join(process.cwd(), 'dist');
|
||||
const distBrowser = path.join(distRoot, 'browser');
|
||||
|
||||
Reference in New Issue
Block a user