feat(youtube): InnerTube-first search, transcripts and watch-next related (Steps 15-18)
- InnerTube layer via pinned youtubei.js 18.1.0 (no quota, no key): search with merged continuations (unlimited pages), watch-next related with LockupView mapping, caption-track discovery - 3-layer dispatcher (YT_SEARCH_MODE, default innertube-first): innertube -> yt-dlp scrape -> official API, graceful errors.yt - Robust yt-dlp binary resolution (YT_DLP_PATH > PATH > bundled) with systematic API fallback (fixes spawn ENOENT in UI) - Transcript: InnerTube caption discovery (YT_TRANSCRIPT_SOURCE), reusing pickTrack/orderedTracks/parseTrackText; yt-dlp fallback kept - Watch: sidebar uses real watch-next related[] (/api/details), title-search fallback for other providers - Cache: memory LRU + SQLite (youtube_search_cache, youtube_metrics), never persist empty pages; /healthz observability; /api/trending - Includes pending Step 15/16 leftovers in same files (suggest, test scripts); unrelated provider adapters left uncommitted
This commit is contained in:
@@ -998,3 +998,74 @@ function subscriptionRowToDto(row) {
|
||||
channel,
|
||||
};
|
||||
}
|
||||
|
||||
// -------------------- Step 17 : cache persistant recherche YouTube --------------------
|
||||
function ensureYoutubeCacheTables() {
|
||||
try {
|
||||
db.exec(`CREATE TABLE IF NOT EXISTS youtube_search_cache (
|
||||
q_hash TEXT PRIMARY KEY, q TEXT NOT NULL, payload_json TEXT NOT NULL,
|
||||
source TEXT NOT NULL DEFAULT 'scrape', created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL
|
||||
);`);
|
||||
db.exec(`CREATE INDEX IF NOT EXISTS idx_yt_cache_exp ON youtube_search_cache(expires_at);`);
|
||||
db.exec(`CREATE TABLE IF NOT EXISTS youtube_metrics (
|
||||
day TEXT PRIMARY KEY, scrape_calls INTEGER NOT NULL DEFAULT 0,
|
||||
api_calls INTEGER NOT NULL DEFAULT 0, quota_units INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL
|
||||
);`);
|
||||
} catch {}
|
||||
}
|
||||
ensureYoutubeCacheTables();
|
||||
|
||||
export function getCachedYoutubeSearch(qHash) {
|
||||
try {
|
||||
ensureYoutubeCacheTables();
|
||||
const row = db.prepare(`SELECT payload_json AS payload, source, expires_at AS exp FROM youtube_search_cache WHERE q_hash = ?`).get(qHash);
|
||||
if (!row) return null;
|
||||
if (Date.now() >= Number(row.exp || 0)) {
|
||||
try { db.prepare(`DELETE FROM youtube_search_cache WHERE q_hash = ?`).run(qHash); } catch {}
|
||||
return null;
|
||||
}
|
||||
try { return { items: JSON.parse(String(row.payload || '[]')), source: row.source }; } catch { return null; }
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
export function setCachedYoutubeSearch(qHash, q, items, source, ttlMs) {
|
||||
try {
|
||||
ensureYoutubeCacheTables();
|
||||
const now = Date.now();
|
||||
db.prepare(`INSERT INTO youtube_search_cache (q_hash, q, payload_json, source, created_at, expires_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(q_hash) DO UPDATE SET q=excluded.q, payload_json=excluded.payload_json,
|
||||
source=excluded.source, created_at=excluded.created_at, expires_at=excluded.expires_at`)
|
||||
.run(qHash, String(q || '').slice(0, 300), JSON.stringify(items || []), source, now, now + ttlMs);
|
||||
// Cap : garde 2000 entrées les plus fraîches
|
||||
try { db.exec(`DELETE FROM youtube_search_cache WHERE q_hash NOT IN (SELECT q_hash FROM youtube_search_cache ORDER BY expires_at DESC LIMIT 2000)`); } catch {}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
export function pruneYoutubeCache() {
|
||||
try { db.prepare(`DELETE FROM youtube_search_cache WHERE expires_at <= ?`).run(Date.now()); } catch {}
|
||||
}
|
||||
|
||||
export function incYoutubeMetrics({ scrapeCalls = 0, apiCalls = 0, quotaUnits = 0 } = {}) {
|
||||
try {
|
||||
ensureYoutubeCacheTables();
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
db.prepare(`INSERT INTO youtube_metrics (day, scrape_calls, api_calls, quota_units, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(day) DO UPDATE SET scrape_calls = scrape_calls + ?, api_calls = api_calls + ?,
|
||||
quota_units = quota_units + ?, updated_at = excluded.updated_at`)
|
||||
.run(day, scrapeCalls, apiCalls, quotaUnits, new Date().toISOString(), scrapeCalls, apiCalls, quotaUnits);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
export function getYoutubeMetricsToday() {
|
||||
try {
|
||||
ensureYoutubeCacheTables();
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
return db.prepare(`SELECT * FROM youtube_metrics WHERE day = ?`).get(day) || { day, scrape_calls: 0, api_calls: 0, quota_units: 0 };
|
||||
} catch { return { scrape_calls: 0, api_calls: 0, quota_units: 0 }; }
|
||||
}
|
||||
|
||||
export function countYoutubeCacheRows() {
|
||||
try { return db.prepare(`SELECT COUNT(1) AS n FROM youtube_search_cache`).get()?.n || 0; } catch { return 0; }
|
||||
}
|
||||
|
||||
+458
-3
@@ -7,12 +7,16 @@ import bcrypt from 'bcryptjs';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import youtubedl from 'youtube-dl-exec';
|
||||
import youtubedlPkg, { create as createYtDlp } from 'youtube-dl-exec';
|
||||
import { execFile as execFileCb } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { fileURLToPath as serverFileURLToPath } from 'node:url';
|
||||
import ffmpegPath from 'ffmpeg-static';
|
||||
import * as cheerio from 'cheerio';
|
||||
import axios from 'axios';
|
||||
import rumbleRouter from './rumble.mjs';
|
||||
import { providerRegistry, validateProviders } from './providers/registry.mjs';
|
||||
import { pickTrack, parseTrackText, parseVtt, orderedTracks, normalizeTranscriptProvider, transcriptTrackExt, looksLikeHtmlError } from './transcript.mjs';
|
||||
import {
|
||||
getUserByUsername,
|
||||
getUserById,
|
||||
@@ -73,9 +77,53 @@ import {
|
||||
} from './db.mjs';
|
||||
import { getChannelAdapter, setTwitchTokenProvider } from './providers/channel-registry.mjs';
|
||||
import { fetchChannelContent } from './providers/channel-content.mjs';
|
||||
import { getSearchMode, getYtDlpBin, hasCookiesFile, metricsSnapshot } from './providers/youtube-common.mjs';
|
||||
import { ytScrapeCacheStats } from './providers/youtube.mjs';
|
||||
|
||||
const app = express();
|
||||
const PORT = Number(process.env.PORT || 4000);
|
||||
// yt-dlp: prefer the newest binary available. The copy bundled with
|
||||
// youtube-dl-exec goes stale (YouTube then answers "The page needs to be
|
||||
// reloaded" to dump-single-json), while a system install is usually fresher.
|
||||
// `YT_DLP_PATH` wins, then `yt-dlp` on PATH, then the bundled binary.
|
||||
const execFileAsync = promisify(execFileCb);
|
||||
async function binaryVersion(bin) {
|
||||
try {
|
||||
const { stdout } = await execFileAsync(bin, ['--version'], { timeout: 15000 });
|
||||
return String(stdout || '').trim().split('\n')[0].trim();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
let youtubedl = youtubedlPkg;
|
||||
let ytDlpInfo = 'bundled';
|
||||
try {
|
||||
let bundledBin = null;
|
||||
try {
|
||||
const u = new URL('../node_modules/youtube-dl-exec/bin/', import.meta.url);
|
||||
const cand = path.join(String(serverFileURLToPath(u)), process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp');
|
||||
if (fs.existsSync(cand)) bundledBin = cand;
|
||||
} catch {}
|
||||
const bundledVer = bundledBin ? await binaryVersion(bundledBin) : null;
|
||||
const candidates = [];
|
||||
if (process.env.YT_DLP_PATH) candidates.push(process.env.YT_DLP_PATH);
|
||||
candidates.push(process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp', 'yt-dlp');
|
||||
let best = null;
|
||||
for (const cand of candidates) {
|
||||
if (!cand) continue;
|
||||
const ver = await binaryVersion(cand);
|
||||
if (ver && (!best || ver > best.ver)) best = { bin: cand, ver };
|
||||
}
|
||||
if (best && (!bundledVer || best.ver >= bundledVer)) {
|
||||
youtubedl = createYtDlp(best.bin);
|
||||
ytDlpInfo = `${best.bin} (${best.ver})`;
|
||||
} else if (bundledVer) {
|
||||
ytDlpInfo = `bundled (${bundledVer})`;
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[config] yt-dlp binary selection failed, using bundled:', e?.message || e);
|
||||
}
|
||||
console.log(`[config] yt-dlp: ${ytDlpInfo}`);
|
||||
const IS_PROD = String(process.env.NODE_ENV || '').toLowerCase() === 'production';
|
||||
const JWT_SECRET = process.env.JWT_SECRET || 'dev-secret-change-me';
|
||||
if (!process.env.JWT_SECRET) {
|
||||
@@ -1006,6 +1054,14 @@ r.get('/peertube/:instance/*', async (req, res) => {
|
||||
});
|
||||
|
||||
// -------------------- Generic video details (GET) --------------------
|
||||
// Cache mémoire pour les vidéos connexes InnerTube (watch-next) : TTL 1h, LRU 200.
|
||||
const YT_RELATED_TTL_MS = Number(process.env.YT_RELATED_TTL_MS || 60 * 60 * 1000);
|
||||
const ytRelatedCache = new Map();
|
||||
function ytRelatedCacheSet(key, items) {
|
||||
if (ytRelatedCache.has(key)) ytRelatedCache.delete(key);
|
||||
ytRelatedCache.set(key, { ts: Date.now(), items });
|
||||
while (ytRelatedCache.size > 200) { const o = ytRelatedCache.keys().next().value; if (o === undefined) break; ytRelatedCache.delete(o); }
|
||||
}
|
||||
// Returns metadata such as title, description, uploader, thumbnail, duration and views for a provider/videoId
|
||||
// Supports query params similar to download endpoints: instance (PeerTube), slug (Odysee), sourceUrl (direct)
|
||||
r.get('/details/:provider/:videoId', async (req, res) => {
|
||||
@@ -1071,6 +1127,22 @@ r.get('/details/:provider/:videoId', async (req, res) => {
|
||||
url,
|
||||
type: 'video',
|
||||
};
|
||||
// Step 18 : vidéos connexes façon SmartTube (watch-next InnerTube, best-effort, 0 quota).
|
||||
// N'implique que YouTube ; toute erreur -> `related: []`, la réponse reste 200.
|
||||
if (String(provider) === 'youtube' && req.query.related !== '0') {
|
||||
try {
|
||||
const { getRelatedViaInnerTube } = await import('./providers/youtube-innertube.mjs');
|
||||
const relKey = `related:${videoId}`;
|
||||
const cached = ytRelatedCache.get(relKey);
|
||||
if (cached && (Date.now() - cached.ts) < YT_RELATED_TTL_MS) {
|
||||
out.related = cached.items;
|
||||
} else {
|
||||
const items = await getRelatedViaInnerTube(videoId, 24).catch(() => []);
|
||||
out.related = items;
|
||||
ytRelatedCacheSet(relKey, items);
|
||||
}
|
||||
} catch { out.related = []; }
|
||||
}
|
||||
return res.json(out);
|
||||
} catch (e) {
|
||||
return res.status(500).json({ error: 'details_failed', details: String(e?.message || e) });
|
||||
@@ -1684,7 +1756,7 @@ r.post('/telemetry/events', authMiddleware, telemetryLimiter, (req, res) => {
|
||||
const { event, meta } = req.body || {};
|
||||
if (!event || typeof event !== 'string') return res.status(400).json({ error: 'event_required' });
|
||||
// Whitelist known event names to keep the table clean
|
||||
const allowed = new Set(['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open']);
|
||||
const allowed = new Set(['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open', 'suggest_shown', 'suggest_used']);
|
||||
if (!allowed.has(event)) return res.status(400).json({ error: 'unknown_event' });
|
||||
const row = insertTelemetryEvent({ userId: req.user.id, event, meta: (meta && typeof meta === 'object') ? meta : null });
|
||||
return res.status(201).json(row || { ok: true });
|
||||
@@ -1704,7 +1776,7 @@ r.get('/telemetry/events', authMiddleware, (req, res) => {
|
||||
r.get('/telemetry/summary', authMiddleware, (req, res) => {
|
||||
const since = typeof req.query.since === 'string' ? req.query.since : undefined;
|
||||
const events = {};
|
||||
for (const name of ['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open']) {
|
||||
for (const name of ['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open', 'suggest_shown', 'suggest_used']) {
|
||||
events[name] = countTelemetryEvents({ event: name, since });
|
||||
}
|
||||
return res.json({ events });
|
||||
@@ -1944,6 +2016,53 @@ r.get('/img/odysee', async (req, res) => {
|
||||
app.use('/api', r);
|
||||
// Health endpoint for container checks
|
||||
app.get('/api/health', (_req, res) => res.json({ status: 'ok' }));
|
||||
// Step 17 : observabilité YouTube (mode, yt-dlp, cache, quota). Aucun secret exposé.
|
||||
app.get(['/healthz', '/api/healthz'], async (_req, res) => {
|
||||
try {
|
||||
const [{ getYoutubeMetricsToday, countYoutubeCacheRows }, common] =
|
||||
await Promise.all([import('./db.mjs'), import('./providers/youtube-common.mjs')]);
|
||||
let ytdlpVersion = null;
|
||||
let resolvedBin = null;
|
||||
try {
|
||||
resolvedBin = await common.resolveYtDlpBin();
|
||||
const { stdout } = await execFileAsync(resolvedBin, ['--version'], { timeout: 10000 });
|
||||
ytdlpVersion = String(stdout || '').trim().split('\n')[0].trim() || null;
|
||||
} catch {}
|
||||
const keys = common.getYouTubeKeys();
|
||||
const banned = [];
|
||||
try { for (const [k, until] of ytKeyBans || []) if (Date.now() < until) banned.push(`...${String(k).slice(-4)}`); } catch {}
|
||||
res.json({
|
||||
status: 'ok',
|
||||
youtube: {
|
||||
mode: getSearchMode(),
|
||||
ytdlp: { bin: resolvedBin || getYtDlpBin(), version: ytdlpVersion, info: ytDlpInfo, binOk: Boolean(ytdlpVersion) },
|
||||
antiban: {
|
||||
cookiesFile: hasCookiesFile(),
|
||||
poToken: Boolean(String(process.env.YT_PO_TOKEN || '').trim()),
|
||||
egressProxy: Boolean(String(process.env.YT_EGRESS_PROXY || '').trim()),
|
||||
},
|
||||
cache: { ...ytScrapeCacheStats(), sqliteRows: countYoutubeCacheRows() },
|
||||
metrics: { ...metricsSnapshot(), today: getYoutubeMetricsToday() },
|
||||
keys: { count: keys.length, banned },
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
res.status(500).json({ status: 'error', error: String(e?.message || e) });
|
||||
}
|
||||
});
|
||||
// Step 17 : trending YouTube sans clé (scrape) avec fallback [] propre.
|
||||
app.get('/api/trending', async (req, res) => {
|
||||
try {
|
||||
const provider = String(req.query.provider || 'yt');
|
||||
const limit = Math.min(50, Math.max(1, Number(req.query.limit || 24)));
|
||||
if (provider !== 'yt') return res.status(400).json({ error: 'only yt supported in phase 1' });
|
||||
const { getTrendingViaScrape } = await import('./providers/youtube-scrape.mjs');
|
||||
const items = await getTrendingViaScrape(limit);
|
||||
return res.json({ provider, items });
|
||||
} catch (e) {
|
||||
return res.json({ provider: 'yt', items: [], error: String(e?.code || e?.message || 'trending_failed') });
|
||||
}
|
||||
});
|
||||
// Alias to support Angular dev proxy paths in both dev and production builds
|
||||
app.use('/proxy/api', r);
|
||||
// Mount dedicated Rumble router (browse, search, video)
|
||||
@@ -2170,6 +2289,342 @@ app.get('/api/search', async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// -------------------- Query typeahead suggestions (Step 15) --------------------
|
||||
// GET /api/search/suggest?q=…&providers=yt,dm&limit=10 -> { q, groups: { yt: string[], dm: string[] } }
|
||||
// Fan-out over provider `suggest()` handlers; providers without one degrade to [] (never 500).
|
||||
const SUGGEST_CACHE_TTL_MS = Number(process.env.SUGGEST_CACHE_TTL_MS || 5 * 60 * 1000);
|
||||
const SUGGEST_CACHE_MAX_ENTRIES = 500;
|
||||
/** @type {Map<string, { ts: number, data: any }>} */
|
||||
const suggestCache = new Map();
|
||||
function suggestCacheGet(key) {
|
||||
const hit = suggestCache.get(key);
|
||||
if (!hit) return null;
|
||||
if ((Date.now() - hit.ts) >= SUGGEST_CACHE_TTL_MS) {
|
||||
suggestCache.delete(key);
|
||||
return null;
|
||||
}
|
||||
// LRU refresh
|
||||
suggestCache.delete(key);
|
||||
suggestCache.set(key, hit);
|
||||
return hit.data;
|
||||
}
|
||||
function suggestCacheSet(key, data) {
|
||||
if (suggestCache.has(key)) suggestCache.delete(key);
|
||||
suggestCache.set(key, { ts: Date.now(), data });
|
||||
while (suggestCache.size > SUGGEST_CACHE_MAX_ENTRIES) {
|
||||
const oldest = suggestCache.keys().next().value;
|
||||
suggestCache.delete(oldest);
|
||||
}
|
||||
}
|
||||
const suggestLimiter = rateLimit({
|
||||
windowMs: 60 * 1000,
|
||||
max: Number(process.env.SUGGEST_RATE_LIMIT || 60),
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
});
|
||||
app.get('/api/search/suggest', suggestLimiter, async (req, res) => {
|
||||
try {
|
||||
const rawQ = typeof req.query.q === 'string' ? req.query.q : '';
|
||||
const q = rawQ.trim();
|
||||
if (q.length < 2) {
|
||||
return res.status(400).json({ error: 'q is required and must be at least 2 characters long' });
|
||||
}
|
||||
const limit = Math.min(20, Math.max(1, Number(req.query.limit || 10)));
|
||||
const requested = typeof req.query.providers === 'string' ? String(req.query.providers) : '';
|
||||
const validProviders = validateProviders(requested);
|
||||
const cacheKey = `suggest:${validProviders.join(',')}:${q.toLowerCase()}:${limit}`;
|
||||
const cached = suggestCacheGet(cacheKey);
|
||||
if (cached) return res.json(cached);
|
||||
const results = await Promise.allSettled(
|
||||
validProviders.map((providerId) => {
|
||||
const mod = providerRegistry[providerId];
|
||||
if (!mod || typeof mod.suggest !== 'function') return Promise.resolve([]);
|
||||
return Promise.resolve().then(() => mod.suggest(q, { limit }));
|
||||
})
|
||||
);
|
||||
const groups = {};
|
||||
results.forEach((result, index) => {
|
||||
const providerId = validProviders[index];
|
||||
if (result.status === 'fulfilled' && Array.isArray(result.value)) {
|
||||
groups[providerId] = result.value
|
||||
.map((s) => String(s ?? '').trim())
|
||||
.filter(Boolean)
|
||||
.slice(0, limit);
|
||||
} else {
|
||||
groups[providerId] = [];
|
||||
}
|
||||
});
|
||||
const data = { q, groups };
|
||||
suggestCacheSet(cacheKey, data);
|
||||
return res.json(data);
|
||||
} catch (e) {
|
||||
return res.status(500).json({ error: 'suggest_failed', details: String(e?.message || e) });
|
||||
}
|
||||
});
|
||||
|
||||
// -------------------- Video transcripts (Step 16, Phase 1) --------------------
|
||||
// GET /api/transcript/:provider/:videoId?lang=&instance=&slug=&sourceUrl=
|
||||
// -> { lang, available, languages, lines: [{ t, dur, text }] }
|
||||
// One endpoint / one parser / one UI whatever the provider (yt-dlp subtitles).
|
||||
const TRANSCRIPT_CACHE_TTL_MS = Number(process.env.TRANSCRIPT_CACHE_TTL || 24 * 60 * 60 * 1000);
|
||||
const TRANSCRIPT_CACHE_MAX_ENTRIES = 200;
|
||||
/** @type {Map<string, { ts: number, data: any }>} */
|
||||
const transcriptCache = new Map();
|
||||
function transcriptCacheGet(key) {
|
||||
const hit = transcriptCache.get(key);
|
||||
if (!hit) return null;
|
||||
if ((Date.now() - hit.ts) >= TRANSCRIPT_CACHE_TTL_MS) {
|
||||
transcriptCache.delete(key);
|
||||
return null;
|
||||
}
|
||||
transcriptCache.delete(key);
|
||||
transcriptCache.set(key, hit);
|
||||
return hit.data;
|
||||
}
|
||||
function transcriptCacheSet(key, data) {
|
||||
if (transcriptCache.has(key)) transcriptCache.delete(key);
|
||||
transcriptCache.set(key, { ts: Date.now(), data });
|
||||
while (transcriptCache.size > TRANSCRIPT_CACHE_MAX_ENTRIES) {
|
||||
const oldest = transcriptCache.keys().next().value;
|
||||
transcriptCache.delete(oldest);
|
||||
}
|
||||
}
|
||||
const transcriptLimiter = rateLimit({
|
||||
windowMs: 60 * 1000,
|
||||
max: Number(process.env.TRANSCRIPT_RATE_LIMIT || 10),
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
// Always answer JSON (default handler sends an HTML/text page, which the
|
||||
// Angular HttpClient cannot parse as JSON and surfaces as a raw SyntaxError).
|
||||
handler: (req, res) => {
|
||||
return res.status(429).json({ available: false, error: 'rate_limited' });
|
||||
},
|
||||
});
|
||||
|
||||
/** Providers known NOT to expose subtitle tracks via yt-dlp (no transcript possible). */
|
||||
const TRANSCRIPT_UNSUPPORTED_PROVIDERS = new Set(['twitch', 'odysee', 'rumble']);
|
||||
|
||||
/** Download subtitles via yt-dlp (handles YouTube impersonation + 429-prone
|
||||
* translated tracks). Tries `langs` in order, returns the first non-empty
|
||||
* parsed lines with the language that worked, or null. Bounded by a timeout
|
||||
* (yt-dlp subtitle downloads can hang on .part files); partial results on
|
||||
* disk are still scanned when yt-dlp exits non-zero. */
|
||||
async function transcriptViaYtDlp(url, langs) {
|
||||
const osMod = await import('node:os');
|
||||
const dir = await fs.promises.mkdtemp(path.join(osMod.tmpdir(), 'newtube-transcript-'));
|
||||
const scanDir = async (wanted) => {
|
||||
let files = [];
|
||||
try {
|
||||
files = (await fs.promises.readdir(dir)).filter((f) => /\.vtt$/i.test(f) && !/\.part$/i.test(f));
|
||||
} catch { return null; }
|
||||
// Ignore stale/empty files
|
||||
const nonEmpty = [];
|
||||
for (const f of files) {
|
||||
try {
|
||||
const st = await fs.promises.stat(path.join(dir, f));
|
||||
if (st.size > 0) nonEmpty.push(f);
|
||||
} catch {}
|
||||
}
|
||||
// Prefer requested languages first
|
||||
nonEmpty.sort((a, b) => {
|
||||
const la = a.toLowerCase(), lb = b.toLowerCase();
|
||||
const ia = wanted.findIndex((w) => la.includes(`.${w.toLowerCase()}.`) || la.endsWith(`.${w.toLowerCase()}.vtt`));
|
||||
const ib = wanted.findIndex((w) => lb.includes(`.${w.toLowerCase()}.`) || lb.endsWith(`.${w.toLowerCase()}.vtt`));
|
||||
return (ia === -1 ? 99 : ia) - (ib === -1 ? 99 : ib);
|
||||
});
|
||||
for (const file of nonEmpty) {
|
||||
try {
|
||||
const text = await fs.promises.readFile(path.join(dir, file), 'utf8');
|
||||
const lines = parseVtt(text);
|
||||
if (lines.length) {
|
||||
const m = /\.([a-z]{2,3}(?:-[a-z]{2,4})?)\.vtt$/i.exec(file);
|
||||
return { lines, lang: m ? m[1] : null };
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
try {
|
||||
const wanted = Array.from(new Set((langs || []).map((l) => String(l || '').split('-')[0]).filter(Boolean)));
|
||||
if (!wanted.includes('en')) wanted.push('en');
|
||||
const subLangs = wanted.slice(0, 4).join(',');
|
||||
const child = youtubedl(url, {
|
||||
writeSub: true,
|
||||
writeAutoSub: true,
|
||||
subLangs,
|
||||
subFormat: 'vtt/best',
|
||||
skipDownload: true,
|
||||
noWarnings: true,
|
||||
noCheckCertificates: true,
|
||||
noPlaylist: true,
|
||||
output: path.join(dir, '%(id)s'),
|
||||
});
|
||||
const timer = setTimeout(() => { try { child.kill('SIGKILL'); } catch {} }, 90000);
|
||||
try {
|
||||
await child;
|
||||
} catch (e) {
|
||||
// Non-zero exit (e.g. one language 429'd) — partial files may still exist.
|
||||
console.warn('[transcript] yt-dlp subtitle download exited non-zero:', String(e?.message || e).slice(0, 200));
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
return await scanDir(wanted);
|
||||
} catch (e) {
|
||||
console.warn('[transcript] yt-dlp subtitle download failed:', e?.message || e);
|
||||
try {
|
||||
const wanted = Array.from(new Set((langs || []).map((l) => String(l || '').split('-')[0]).filter(Boolean)));
|
||||
return await scanDir(wanted);
|
||||
} catch { return null; }
|
||||
} finally {
|
||||
try { await fs.promises.rm(dir, { recursive: true, force: true }); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
/** Fetch one timedtext track URL and parse it (any format). Returns [] on any failure. */
|
||||
async function fetchTimedTextLines(track) {
|
||||
const resp = await fetch(String(track.url), {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
|
||||
Accept: 'application/json, text/vtt, text/*;q=0.9, */*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9,fr;q=0.8',
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`track_fetch_failed:${resp.status}`);
|
||||
const text = await resp.text();
|
||||
if (!text || looksLikeHtmlError(text)) throw new Error('track_fetch_failed:html_error_page');
|
||||
return parseTrackText(text, transcriptTrackExt(track));
|
||||
}
|
||||
|
||||
app.get('/api/transcript/:provider/:videoId', transcriptLimiter, async (req, res) => {
|
||||
try {
|
||||
const { provider, videoId } = req.params;
|
||||
const lang = String(req.query.lang || 'fr').slice(0, 12) || 'fr';
|
||||
const normalized = normalizeTranscriptProvider(provider);
|
||||
if (!normalized || !videoId) {
|
||||
return res.status(400).json({ available: false, error: 'invalid_provider_or_video' });
|
||||
}
|
||||
const cacheKey = `transcript:${normalized}:${videoId}:${lang.toLowerCase()}`;
|
||||
const cached = transcriptCacheGet(cacheKey);
|
||||
if (cached) return res.json(cached);
|
||||
// Source de découverte des pistes : InnerTube d'abord pour YouTube
|
||||
// (mêmes URLs timedtext, sans spawn yt-dlp), yt-dlp sinon/en secours.
|
||||
// YT_TRANSCRIPT_SOURCE=innertube-first (défaut) | ytdlp-only | innertube-only
|
||||
const transcriptSource = String(process.env.YT_TRANSCRIPT_SOURCE || 'innertube-first').trim().toLowerCase();
|
||||
const transcriptUrl = providerUrlFrom(normalized, String(videoId), {
|
||||
instance: req.query.instance || undefined,
|
||||
slug: req.query.slug || undefined,
|
||||
sourceUrl: req.query.sourceUrl || undefined,
|
||||
});
|
||||
let meta = null;
|
||||
if (normalized === 'youtube' && transcriptSource !== 'ytdlp-only') {
|
||||
try {
|
||||
const { getCaptionTracksViaInnerTube } = await import('./providers/youtube-innertube.mjs');
|
||||
const cap = await getCaptionTracksViaInnerTube(String(videoId));
|
||||
if (cap.trackCount > 0) {
|
||||
meta = { subtitles: cap.subtitles, automatic_captions: cap.automatic_captions };
|
||||
console.log(`[transcript] source=innertube tracks=${cap.trackCount} langs=${cap.languages.join(',')}`);
|
||||
} else {
|
||||
// InnerTube fait foi (même backend que le lecteur) : 0 piste = pas
|
||||
// de sous-titres, sans payer un dump yt-dlp complet.
|
||||
console.log('[transcript] source=innertube tracks=0 -> no_subtitles');
|
||||
const empty = { lang: null, available: false, languages: [], lines: [], reason: 'no_subtitles' };
|
||||
transcriptCacheSet(cacheKey, empty);
|
||||
return res.json(empty);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[transcript] innertube discovery failed, fallback yt-dlp:', e?.code || String(e?.message || e).slice(0, 120));
|
||||
if (transcriptSource === 'innertube-only') {
|
||||
return res.status(502).json({ available: false, languages: [], lines: [], error: 'transcript_temporarily_unavailable', retryable: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!meta) {
|
||||
try {
|
||||
const raw = await youtubedl(transcriptUrl, { dumpSingleJson: true, skipDownload: true, noWarnings: true, noCheckCertificates: true });
|
||||
meta = (typeof raw === 'string') ? JSON.parse(raw || '{}') : (raw || {});
|
||||
} catch (e) {
|
||||
console.error('[transcript] yt-dlp failed:', e?.message || e);
|
||||
return res.status(502).json({ available: false, languages: [], lines: [], error: 'transcript_temporarily_unavailable', retryable: true });
|
||||
}
|
||||
}
|
||||
const { track, languages, lang: chosenLang } = pickTrack(meta, lang);
|
||||
if (!track) {
|
||||
// Definitive absence: distinguish "provider never exposes subtitles"
|
||||
// (twitch/odysee/rumble) from "this video has none" — cacheable 200s.
|
||||
const reason = TRANSCRIPT_UNSUPPORTED_PROVIDERS.has(normalized) ? 'provider_unsupported' : 'no_subtitles';
|
||||
const empty = { lang: null, available: false, languages: languages || [], lines: [], reason };
|
||||
transcriptCacheSet(cacheKey, empty);
|
||||
return res.json(empty);
|
||||
}
|
||||
// Try candidate tracks in order: requested language, then original (`en`),
|
||||
// then everything else. Translated tracks are often rate-limited while the
|
||||
// original still works — never fail on the first track alone.
|
||||
// Bounded: hammering dozens of timedtext URLs only worsens YouTube 429s.
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
let lines = [];
|
||||
let workedLang = chosenLang;
|
||||
let sawRateLimit = false;
|
||||
for (const cand of orderedTracks(meta, lang).slice(0, 5)) {
|
||||
let attempt = 0;
|
||||
for (;;) {
|
||||
try {
|
||||
const parsed = await fetchTimedTextLines(cand.track);
|
||||
if (parsed && parsed.length) {
|
||||
lines = parsed;
|
||||
workedLang = cand.lang || chosenLang;
|
||||
}
|
||||
break;
|
||||
} catch (e) {
|
||||
const msg = String(e?.message || e);
|
||||
console.warn('[transcript] track fetch failed:', cand.lang, msg);
|
||||
// One retry after a short pause on transient 429s.
|
||||
if (msg.includes(':429') && attempt === 0) {
|
||||
sawRateLimit = true;
|
||||
attempt += 1;
|
||||
await sleep(1500);
|
||||
continue;
|
||||
}
|
||||
if (msg.includes(':429')) sawRateLimit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lines.length) break;
|
||||
}
|
||||
if (!lines || lines.length === 0) {
|
||||
// Direct timedtext fetches failed (429 / Sorry pages / impersonation):
|
||||
// let yt-dlp download the subtitles instead (handles all of the above).
|
||||
try {
|
||||
const viaDlp = await transcriptViaYtDlp(transcriptUrl, [lang, chosenLang].filter(Boolean));
|
||||
if (viaDlp && viaDlp.lines && viaDlp.lines.length) {
|
||||
lines = viaDlp.lines;
|
||||
if (viaDlp.lang) workedLang = viaDlp.lang;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
if (!lines || lines.length === 0) {
|
||||
// Subtitle tracks exist but their content could not be retrieved or
|
||||
// parsed (YouTube 429 / Sorry pages / impersonation): the video HAS
|
||||
// subtitles, so this is transient — 502 with languages + retryable flag,
|
||||
// never cached as "no subtitles".
|
||||
return res.status(502).json({
|
||||
available: false,
|
||||
languages: languages || [],
|
||||
lines: [],
|
||||
error: 'transcript_temporarily_unavailable',
|
||||
retryable: true,
|
||||
rateLimited: sawRateLimit || undefined,
|
||||
});
|
||||
}
|
||||
const result = { lang: workedLang, available: true, languages: languages || [], lines };
|
||||
transcriptCacheSet(cacheKey, result);
|
||||
return res.json(result);
|
||||
} catch (e) {
|
||||
console.error('[transcript] unexpected error:', e?.message || e);
|
||||
return res.status(502).json({ available: false, languages: [], lines: [], error: 'transcript_temporarily_unavailable', retryable: true });
|
||||
}
|
||||
});
|
||||
|
||||
// -------------------- Static Frontend (Angular build) --------------------
|
||||
const distRoot = path.join(process.cwd(), 'dist');
|
||||
const distBrowser = path.join(distRoot, 'browser');
|
||||
|
||||
@@ -63,6 +63,16 @@ async function resolveYouTubeChannelId(externalId) {
|
||||
const raw = String(externalId || '').trim();
|
||||
if (/^UC[\w-]{20,}$/.test(raw)) return raw;
|
||||
const handle = raw.replace(/^@/, '');
|
||||
// 0) scrape sans clé (Step 17) : ne consomme aucun quota
|
||||
try {
|
||||
const { getSearchMode } = await import('./youtube-common.mjs');
|
||||
const mode = getSearchMode();
|
||||
if (mode !== 'api-only') {
|
||||
const { resolveChannelIdViaScrape } = await import('./youtube-scrape.mjs');
|
||||
const id = await resolveChannelIdViaScrape(raw);
|
||||
if (id && /^UC[\w-]{20,}$/.test(id)) return id;
|
||||
}
|
||||
} catch {}
|
||||
// 1) channels?forHandle (fonctionne encore pour beaucoup de chaînes)
|
||||
try {
|
||||
const data = await ytGet('channels', { part: 'id', forHandle: handle });
|
||||
@@ -131,9 +141,32 @@ function ytTokenFor(key, page) {
|
||||
}
|
||||
|
||||
async function ytContent(externalId, { type, page, limit, sort, q }) {
|
||||
const channelId = await resolveYouTubeChannelId(externalId);
|
||||
const perPage = Math.min(Math.max(1, Number(limit || 24)), 50);
|
||||
const pageNum = Math.max(1, Number(page || 1));
|
||||
// Step 17 : scrape-first sans clé (0 quota). Fallback API si bot-check/timeout.
|
||||
try {
|
||||
const { getSearchMode } = await import('./youtube-common.mjs');
|
||||
const mode = getSearchMode();
|
||||
if (mode !== 'api-only') {
|
||||
const { fetchChannelViaScrape } = await import('./youtube-scrape.mjs');
|
||||
// externalId brut (handle ou UC...) : le scrape gère les deux formes
|
||||
const scraped = await fetchChannelViaScrape(externalId, { type, page: pageNum, limit: perPage });
|
||||
if (Array.isArray(scraped?.items) && scraped.items.length) {
|
||||
if (mode === 'scrape-only') return { ...scraped, total: null };
|
||||
// scrape-first : retour direct si non vide
|
||||
return { ...scraped, total: null };
|
||||
}
|
||||
// vide -> on tente l'API (chaîne à faible volume ou tab non supporté en scrape)
|
||||
if (mode === 'scrape-only') return { items: [], nextPage: null };
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[channel-content/yt] scrape failed, fallback api:', e?.code || e?.message || e);
|
||||
try {
|
||||
const { getSearchMode } = await import('./youtube-common.mjs');
|
||||
if (getSearchMode() === 'scrape-only') return { items: [], nextPage: null };
|
||||
} catch {}
|
||||
}
|
||||
const channelId = await resolveYouTubeChannelId(externalId);
|
||||
if (type === 'playlists') {
|
||||
const key = ['pl', channelId, perPage].join('|');
|
||||
const token = ytTokenFor(key, pageNum);
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
// Helpers partagés YouTube : clés API, mode de recherche, anti-ban yt-dlp, métriques.
|
||||
// Extrait de youtube.mjs + index.mjs pour éviter la duplication (Step 17 P0).
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { execFile as execFileCb } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
export function getYouTubeKeys() {
|
||||
const keys = [];
|
||||
try {
|
||||
const raw = process.env.YOUTUBE_API_KEYS;
|
||||
if (raw && String(raw).trim() && !['undefined', 'null'].includes(String(raw).trim())) {
|
||||
const s = String(raw).trim();
|
||||
if (s.startsWith('[')) {
|
||||
try {
|
||||
const arr = JSON.parse(s);
|
||||
if (Array.isArray(arr)) keys.push(...arr.map((v) => String(v || '').trim()).filter(Boolean));
|
||||
} catch {}
|
||||
} else {
|
||||
keys.push(...s.split(',').map((v) => String(v || '').trim()).filter(Boolean));
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
try {
|
||||
const single = process.env.YOUTUBE_API_KEY;
|
||||
if (single && String(single).trim()) keys.push(String(single).trim());
|
||||
} catch {}
|
||||
return Array.from(new Set(keys.filter(Boolean)));
|
||||
}
|
||||
|
||||
export function isKeyFailure(status, data) {
|
||||
try {
|
||||
const reason = data?.error?.errors?.[0]?.reason || '';
|
||||
const message = String(data?.error?.message || '');
|
||||
if (status === 400 && (reason === 'API_KEY_INVALID' || /api key (expired|invalid)/i.test(message))) return true;
|
||||
if (status === 403 && /quota|rateLimit|dailyLimit|userRateLimit/i.test(`${reason} ${message}`)) return true;
|
||||
} catch {}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Mode de recherche YT : innertube-first (défaut) | innertube-only | scrape-first | api-first | scrape-only | api-only */
|
||||
export const YT_SEARCH_MODES = ['innertube-first', 'innertube-only', 'scrape-first', 'api-first', 'scrape-only', 'api-only'];
|
||||
export function getSearchMode() {
|
||||
const m = String(process.env.YT_SEARCH_MODE || 'innertube-first').trim().toLowerCase();
|
||||
if (YT_SEARCH_MODES.includes(m)) return m;
|
||||
return 'innertube-first';
|
||||
}
|
||||
|
||||
export function getScrapeTtlMs() {
|
||||
const v = Number(process.env.YT_SCRAPE_TTL_MS || 30 * 60 * 1000);
|
||||
return Number.isFinite(v) && v > 0 ? v : 30 * 60 * 1000;
|
||||
}
|
||||
|
||||
export function getYtDlpTimeoutMs() {
|
||||
const v = Number(process.env.YT_DLP_TIMEOUT_MS || 20000);
|
||||
return Number.isFinite(v) && v >= 5000 ? v : 20000;
|
||||
}
|
||||
|
||||
export function getYtDlpBin() {
|
||||
const p = String(process.env.YT_DLP_PATH || '').trim();
|
||||
if (p) return p;
|
||||
return process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp';
|
||||
}
|
||||
|
||||
/**
|
||||
* Résolution robuste du binaire yt-dlp (même ordre que server/index.mjs) :
|
||||
* 1. YT_DLP_PATH (si le fichier existe)
|
||||
* 2. `yt-dlp[.exe]` sur le PATH (probé via --version)
|
||||
* 3. binaire bundled de `youtube-dl-exec` (node_modules/.../bin/, utilisé en Docker)
|
||||
* Résultat mis en cache process-wide. Throw avec code `yt_scrape_no_binary` si introuvable.
|
||||
*/
|
||||
let _resolvedBin = null;
|
||||
export function resetYtDlpBinCache() { _resolvedBin = null; }
|
||||
|
||||
function bundledYtDlpPath() {
|
||||
try {
|
||||
const exe = process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp';
|
||||
// server/providers/ -> server/../node_modules/youtube-dl-exec/bin/
|
||||
const cand = path.resolve(path.dirname(new URL(import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1')), '..', '..', 'node_modules', 'youtube-dl-exec', 'bin', exe);
|
||||
if (fs.existsSync(cand)) return cand;
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function getBundledYtDlpPath() { return bundledYtDlpPath(); }
|
||||
|
||||
async function probeBin(bin) {
|
||||
try {
|
||||
const execFileAsync = promisify(execFileCb);
|
||||
await execFileAsync(bin, ['--version'], { timeout: 10000 });
|
||||
return true;
|
||||
} catch (e) {
|
||||
// ENOENT = binaire absent ; autre erreur (timeout...) = présent mais KO -> on le garde quand même
|
||||
return e?.code !== 'ENOENT' && e?.errno !== 'ENOENT';
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveYtDlpBin() {
|
||||
if (_resolvedBin) return _resolvedBin;
|
||||
// 1) YT_DLP_PATH explicite
|
||||
try {
|
||||
const p = String(process.env.YT_DLP_PATH || '').trim();
|
||||
if (p && fs.existsSync(p)) {
|
||||
if (await probeBin(p)) { _resolvedBin = p; return p; }
|
||||
} else if (p) {
|
||||
console.warn(`[YT] YT_DLP_PATH introuvable : ${p} (fallback PATH/bundled)`);
|
||||
}
|
||||
} catch {}
|
||||
// 2) PATH
|
||||
const exe = process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp';
|
||||
for (const cand of [exe, 'yt-dlp']) {
|
||||
try {
|
||||
if (await probeBin(cand)) { _resolvedBin = cand; return cand; }
|
||||
} catch {}
|
||||
}
|
||||
// 3) bundled youtube-dl-exec
|
||||
const bundled = bundledYtDlpPath();
|
||||
if (bundled && await probeBin(bundled)) { _resolvedBin = bundled; return bundled; }
|
||||
throw Object.assign(
|
||||
new Error('yt-dlp introuvable (PATH, YT_DLP_PATH ni binaire bundled). Installez yt-dlp ou définissez YT_DLP_PATH.'),
|
||||
{ ytStatus: 503, code: 'yt_scrape_no_binary' },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Args anti-ban communs pour yt-dlp (cookies, PO-Token, proxy).
|
||||
* Ne jamais logger les valeurs (cookies path exclu des logs verbeux).
|
||||
*/
|
||||
export function buildYtDlpExtraArgs() {
|
||||
const args = [];
|
||||
try {
|
||||
const cookies = String(process.env.YT_COOKIES_FILE || '').trim();
|
||||
if (cookies && fs.existsSync(cookies)) {
|
||||
args.push('--cookies', cookies);
|
||||
}
|
||||
} catch {}
|
||||
try {
|
||||
const po = String(process.env.YT_PO_TOKEN || '').trim();
|
||||
if (po) args.push('--extractor-args', `youtube:po_token=${po}`);
|
||||
} catch {}
|
||||
try {
|
||||
const proxy = String(process.env.YT_EGRESS_PROXY || '').trim();
|
||||
if (proxy) args.push('--proxy', proxy);
|
||||
} catch {}
|
||||
return args;
|
||||
}
|
||||
|
||||
export function hasCookiesFile() {
|
||||
try {
|
||||
const c = String(process.env.YT_COOKIES_FILE || '').trim();
|
||||
return Boolean(c && fs.existsSync(c));
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
export function hashSearchKey(parts) {
|
||||
return crypto.createHash('sha256').update(String(parts)).digest('hex').slice(0, 32);
|
||||
}
|
||||
|
||||
// Métriques process-wide (remises à zéro au restart, persistées jour par jour en SQLite via db.mjs)
|
||||
export const ytMetrics = {
|
||||
scrapeCalls: 0,
|
||||
scrapeHits: 0, // hit cache (mémoire ou SQLite)
|
||||
apiCalls: 0,
|
||||
quotaUnits: 0, // estimation search*100 + videos*1
|
||||
fallbacks: 0,
|
||||
scrapeErrors: 0,
|
||||
innertubeCalls: 0,
|
||||
innertubeErrors: 0,
|
||||
};
|
||||
|
||||
export function metricsSnapshot() {
|
||||
return { ...ytMetrics };
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
// Couche InnerTube directe (façon SmartTube/MediaServiceCore) via youtubei.js.
|
||||
// WEB client : search + continuations (pagination illimitée), watch-next (related),
|
||||
// sans clé API ni quota. Le dispatcher bascule sur scrape/API si indisponible.
|
||||
import { hashSearchKey, ytMetrics } from './youtube-common.mjs';
|
||||
|
||||
let sessionPromise = null;
|
||||
let LogSilenced = false;
|
||||
|
||||
async function silenceLibNoise() {
|
||||
if (LogSilenced) return;
|
||||
LogSilenced = true;
|
||||
try {
|
||||
const { Log } = await import('youtubei.js');
|
||||
// youtubei.js loggue en WARN chaque Text sans run assorti (bruit sur les titres) :
|
||||
// on ne garde que les erreurs.
|
||||
if (Log?.set_level && Log?.Level) {
|
||||
const lvl = Log.Level.ERROR ?? Log.Level.WARNING ?? 3;
|
||||
Log.set_level(lvl);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
/** Session InnerTube singleton (lazy). Throw code yt_innertube_unavailable si KO. */
|
||||
export async function getSession() {
|
||||
if (!sessionPromise) {
|
||||
sessionPromise = (async () => {
|
||||
try {
|
||||
await silenceLibNoise();
|
||||
const { Innertube } = await import('youtubei.js');
|
||||
const gl = String(process.env.YT_INNERTUBE_GL || 'FR').trim() || 'FR';
|
||||
const hl = String(process.env.YT_INNERTUBE_HL || 'fr').trim() || 'fr';
|
||||
return await Innertube.create({ lang: hl, location: gl });
|
||||
} catch (e) {
|
||||
sessionPromise = null; // retry au prochain appel
|
||||
throw Object.assign(
|
||||
new Error(`InnerTube indisponible : ${String(e?.message || e).slice(0, 160)}`),
|
||||
{ ytStatus: 502, code: 'yt_innertube_unavailable' },
|
||||
);
|
||||
}
|
||||
})();
|
||||
}
|
||||
return sessionPromise;
|
||||
}
|
||||
|
||||
export function resetSession() { sessionPromise = null; }
|
||||
|
||||
/** "81 973 vues" / "57K" / "1,2 M vues" -> number | undefined. */
|
||||
export function parseViewsText(t) {
|
||||
try {
|
||||
const s = String(t || '').trim();
|
||||
if (!s) return undefined;
|
||||
const m = s.match(/([\d\s\u00a0.,]+)\s*([KMBkmb]|Mds|M|k|B)?/);
|
||||
if (!m) return undefined;
|
||||
const num = Number(m[1].replace(/[\s\u00a0]/g, '').replace(',', '.'));
|
||||
if (!Number.isFinite(num)) return undefined;
|
||||
const suffix = (m[2] || '').toLowerCase();
|
||||
const mult = suffix === 'b' ? 1e9 : suffix === 'm' || suffix === 'mds' ? 1e6 : suffix === 'k' ? 1e3 : 1;
|
||||
const v = Math.round(num * mult);
|
||||
return v > 0 ? v : undefined;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
/** "2 minutes, 27 seconds" / "1 heure, 5 minutes" (label a11y) -> secondes. */
|
||||
export function parseDurationLabel(label) {
|
||||
try {
|
||||
const s = String(label || '').toLowerCase();
|
||||
if (!s) return undefined;
|
||||
const get = (re) => { const m = s.match(re); return m ? Number(m[1]) : 0; };
|
||||
const h = get(/(\d+)\s*(?:hours?|heures?)/);
|
||||
const mnt = get(/(\d+)\s*(?:minutes?)/);
|
||||
const sec = get(/(\d+)\s*(?:seconds?|secondes?)/);
|
||||
const total = h * 3600 + mnt * 60 + sec;
|
||||
return total > 0 ? total : undefined;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
function bestThumb(thumbs) {
|
||||
try {
|
||||
const arr = Array.isArray(thumbs) ? thumbs.filter((t) => t?.url) : [];
|
||||
if (!arr.length) return undefined;
|
||||
return arr[arr.length - 1].url;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Mappe un node InnerTube (Video, CompactVideo, GridVideo, LockupView, ReelItem,
|
||||
* ShortsLockupView, PlaylistPanelVideo, WatchCardCompactVideo) vers Suggestion.
|
||||
* Pur et testable offline. Les LockupView (nouveau renderer YouTube, utilisé
|
||||
* notamment dans le watch-next façon SmartTube) ont une forme imbriquée propre.
|
||||
*/
|
||||
export function mapVideoNode(n) {
|
||||
if (!n || typeof n !== 'object') return null;
|
||||
try {
|
||||
const type = String(n.type || '');
|
||||
if (type === 'LockupView') return mapLockupView(n);
|
||||
if (/playlist|channel|gridchannel|shelf|radio|show|album/i.test(type)
|
||||
&& !/playlistpanelvideo|watchcard/i.test(type)) return null;
|
||||
const id = n.id ? String(n.id) : null;
|
||||
if (!id) return null;
|
||||
const title = n.title?.text ?? (typeof n.title === 'string' ? n.title : '');
|
||||
if (!title) return null;
|
||||
const duration = n.duration?.seconds != null ? Number(n.duration.seconds) : undefined;
|
||||
const author = n.author || n.uploader || null;
|
||||
const authorName = author?.name ?? (typeof author === 'string' ? author : undefined);
|
||||
const authorId = author?.id ? String(author.id) : undefined;
|
||||
const views = parseViewsText(n.view_count?.text ?? n.view_count ?? n.views?.text);
|
||||
const isShort = /short|reel/i.test(type) || (Number.isFinite(duration) && duration > 0 && duration <= 70 && /short/i.test(title) === false && /reel|short/i.test(type));
|
||||
const badges = Array.isArray(n.badges) ? n.badges.map((b) => b?.label).filter(Boolean) : [];
|
||||
return {
|
||||
title: String(title),
|
||||
id,
|
||||
url: `https://www.youtube.com/watch?v=${id}`,
|
||||
thumbnail: bestThumb(n.thumbnails),
|
||||
uploaderName: authorName,
|
||||
type: 'video',
|
||||
...(Number.isFinite(duration) && duration > 0 ? { duration } : {}),
|
||||
...(views !== undefined ? { views } : {}),
|
||||
...(n.published?.text ? { publishedAt: String(n.published.text) } : {}),
|
||||
...(authorId ? { channelId: authorId, channelExternalId: authorId, channelUrl: `https://www.youtube.com/channel/${authorId}` } : {}),
|
||||
...(authorName ? { channelHandle: String(authorName) } : {}),
|
||||
...(n.is_live ? { isLive: true } : {}),
|
||||
...(isShort ? { isShort: true } : {}),
|
||||
...(badges.length ? { badges } : {}),
|
||||
};
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
/** Mappe un LockupView (renderer moderne : watch-next, search, shelves). */
|
||||
export function mapLockupView(n) {
|
||||
try {
|
||||
const ct = String(n.content_type || '').toUpperCase();
|
||||
if (ct && !['VIDEO', 'SHORT', 'MOVIE', 'LIVE'].includes(ct)) return null;
|
||||
const id = n.content_id ? String(n.content_id)
|
||||
: (n.renderer_context?.command_context?.on_tap?.payload?.videoId || null);
|
||||
if (!id) return null;
|
||||
const meta = n.metadata || {};
|
||||
const title = meta.title?.text ?? (typeof meta.title === 'string' ? meta.title : '');
|
||||
if (!title) return null;
|
||||
// Vignette : content_image.image[] (prend la plus large)
|
||||
let thumbnail;
|
||||
try {
|
||||
const imgs = (n.content_image?.image || []).filter((i) => i?.url);
|
||||
thumbnail = imgs.sort((a, b) => (b.width || 0) - (a.width || 0))[0]?.url;
|
||||
} catch {}
|
||||
// Chaîne : 1ère ligne des metadata_rows, sinon label a11y "Go to channel X"
|
||||
let uploaderName;
|
||||
try {
|
||||
const rows = meta.metadata?.metadata_rows || [];
|
||||
const first = rows[0]?.metadata_parts?.[0]?.text;
|
||||
uploaderName = first?.text ?? (typeof first === 'string' ? first : undefined);
|
||||
} catch {}
|
||||
if (!uploaderName) {
|
||||
const a11y = String(meta.image?.a11y_label || '');
|
||||
const m = a11y.match(/^(?:go to channel|aller sur la cha[îi]ne)\s+(.+)$/i);
|
||||
if (m) uploaderName = m[1].trim();
|
||||
}
|
||||
// Vues : 2e ligne ("57K", "1,2 M vues"...), durée : label a11y
|
||||
let views;
|
||||
try {
|
||||
const rows = meta.metadata?.metadata_rows || [];
|
||||
const second = rows[1]?.metadata_parts?.[0]?.text;
|
||||
views = parseViewsText(second?.text ?? second);
|
||||
} catch {}
|
||||
const duration = parseDurationLabel(n.renderer_context?.accessibility_context?.label);
|
||||
return {
|
||||
title: String(title),
|
||||
id,
|
||||
url: `https://www.youtube.com/watch?v=${id}`,
|
||||
thumbnail,
|
||||
uploaderName,
|
||||
type: 'video',
|
||||
...(duration !== undefined ? { duration } : {}),
|
||||
...(views !== undefined ? { views } : {}),
|
||||
...(uploaderName ? { channelHandle: String(uploaderName) } : {}),
|
||||
...(ct === 'SHORT' ? { isShort: true } : {}),
|
||||
...(ct === 'LIVE' ? { isLive: true } : {}),
|
||||
};
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
export function mapNodes(nodes) {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
for (const n of nodes || []) {
|
||||
const m = mapVideoNode(n);
|
||||
if (!m || seen.has(m.id)) continue;
|
||||
seen.add(m.id);
|
||||
out.push(m);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Chaîne de continuations en mémoire : q_hash -> { feed, pages }.
|
||||
// (SQLite persiste les pages déjà servies ; la chaîne évite de rejouer les pages 1..N-1.)
|
||||
const chainCache = new Map();
|
||||
const CHAIN_MAX = 50;
|
||||
function chainGet(k) { const h = chainCache.get(k); if (h) { chainCache.delete(k); chainCache.set(k, h); } return h || null; }
|
||||
function chainSet(k, v) {
|
||||
if (chainCache.has(k)) chainCache.delete(k);
|
||||
chainCache.set(k, v);
|
||||
while (chainCache.size > CHAIN_MAX) { const o = chainCache.keys().next().value; if (o === undefined) break; chainCache.delete(o); }
|
||||
}
|
||||
export function innertubeChainStats() { return { chains: chainCache.size, max: CHAIN_MAX }; }
|
||||
|
||||
function sortParam(sort) {
|
||||
const s = String(sort || 'relevance').toLowerCase();
|
||||
if (s === 'date') return 'upload_date';
|
||||
if (s === 'views') return 'view_count';
|
||||
return 'relevance';
|
||||
}
|
||||
|
||||
/**
|
||||
* Recherche InnerTube avec vraie pagination (continuations).
|
||||
* Page 1 ~35-40 vidéos ; pages suivantes via getContinuation() fusionnées (façon SmartTube).
|
||||
*/
|
||||
export async function searchViaInnerTube(q, opts = {}) {
|
||||
const query = String(q || '').trim();
|
||||
if (query.length < 2) return [];
|
||||
const limit = Math.min(50, Math.max(1, Number(opts?.limit || 24)));
|
||||
const page = Math.min(10, Math.max(1, Number(opts?.page || 1)));
|
||||
const sort = String(opts?.sort || 'relevance');
|
||||
ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
|
||||
const yt = await getSession();
|
||||
const key = `it|${hashSearchKey(`${query.toLowerCase()}|${sort}`)}`;
|
||||
let entry = chainGet(key);
|
||||
let feed = entry?.feed || null;
|
||||
let collected = entry?.items ? [...entry.items] : [];
|
||||
let pagesDone = entry?.pages || 0;
|
||||
// Note : une "page" InnerTube fait ~17-20 vidéos quel que soit `limit`.
|
||||
// On charge donc des continuations jusqu'à couvrir la fenêtre demandée
|
||||
// [start, start+limit[ (façon SmartTube qui remplit son écran au fil des
|
||||
// continuations), au lieu d'aligner 1 page API = 1 page UI.
|
||||
const need = page * limit;
|
||||
try {
|
||||
if (!feed) {
|
||||
feed = await yt.search(query, { sort_by: sortParam(sort) });
|
||||
collected = mapNodes(feed.videos);
|
||||
pagesDone = 1;
|
||||
}
|
||||
let guard = 0;
|
||||
while (collected.length < need && feed?.has_continuation && guard < 12) {
|
||||
guard++;
|
||||
feed = await feed.getContinuation();
|
||||
const fresh = mapNodes(feed.videos).filter((m) => !collected.some((c) => c.id === m.id));
|
||||
collected = collected.concat(fresh);
|
||||
pagesDone++;
|
||||
if (!fresh.length) break;
|
||||
}
|
||||
} catch (e) {
|
||||
if (e?.code === 'yt_innertube_unavailable') throw e;
|
||||
throw Object.assign(new Error(`InnerTube search failed: ${String(e?.message || e).slice(0, 160)}`), { ytStatus: 502, code: 'yt_innertube_failed' });
|
||||
}
|
||||
if (feed) chainSet(key, { feed, items: collected, pages: pagesDone });
|
||||
const start = (page - 1) * limit;
|
||||
return collected.slice(start, start + limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Vidéos connexes façon SmartTube (endpoint watch-next) : ce qui alimente
|
||||
* la colonne "À suivre" sous le lecteur.
|
||||
*/
|
||||
export async function getRelatedViaInnerTube(videoId, limit = 24) {
|
||||
const id = String(videoId || '').trim();
|
||||
if (!id) return [];
|
||||
const n = Math.min(50, Math.max(1, Number(limit || 24)));
|
||||
ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
|
||||
const yt = await getSession();
|
||||
try {
|
||||
const info = await yt.getInfo(id);
|
||||
const feed = info?.watch_next_feed;
|
||||
const arr = Array.isArray(feed) ? feed : (feed ? Array.from(feed) : []);
|
||||
return mapNodes(arr).filter((m) => m.id !== id).slice(0, n);
|
||||
} catch (e) {
|
||||
if (e?.code === 'yt_innertube_unavailable') throw e;
|
||||
throw Object.assign(new Error(`InnerTube related failed: ${String(e?.message || e).slice(0, 160)}`), { ytStatus: 502, code: 'yt_innertube_failed' });
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------- Transcripts via InnerTube --------------------
|
||||
// `getInfo().captions.caption_tracks[]` expose les mêmes URLs timedtext que
|
||||
// yt-dlp découvre via un dump complet (`subtitles`/`automatic_captions`).
|
||||
// On adapte leur forme vers le format yt-dlp pour réutiliser pickTrack(),
|
||||
// orderedTracks() et parseTrackText() de transcript.mjs sans les toucher.
|
||||
|
||||
function normCaptionLang(code) {
|
||||
return String(code || '').trim().toLowerCase().replace(/_/g, '-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapte des caption tracks InnerTube brutes vers un pseudo dump yt-dlp
|
||||
* `{ subtitles, automatic_captions }` (kind 'asr' = auto-généré).
|
||||
* Pur et testable offline.
|
||||
*/
|
||||
export function mapCaptionTracks(captionTracks) {
|
||||
const subtitles = {};
|
||||
const automatic_captions = {};
|
||||
for (const t of captionTracks || []) {
|
||||
const base = String(t?.base_url || '');
|
||||
if (!base) continue;
|
||||
const lang = normCaptionLang(t?.language_code) || 'und';
|
||||
const name = t?.name?.text ?? (typeof t?.name === 'string' ? t.name : lang);
|
||||
const sep = base.includes('?') ? '&' : '?';
|
||||
const track = { url: `${base}${sep}fmt=json3`, ext: 'json3', name: String(name) };
|
||||
const dict = t?.kind === 'asr' ? automatic_captions : subtitles;
|
||||
if (!Array.isArray(dict[lang])) dict[lang] = [];
|
||||
if (!dict[lang].some((x) => x.url === track.url)) dict[lang].push(track);
|
||||
}
|
||||
const languages = Array.from(new Set([...Object.keys(subtitles), ...Object.keys(automatic_captions)]));
|
||||
const trackCount = Object.values(subtitles).concat(Object.values(automatic_captions))
|
||||
.reduce((n, arr) => n + arr.length, 0);
|
||||
return { subtitles, automatic_captions, languages, trackCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* Découvre les pistes de sous-titres d'une vidéo via InnerTube (0 quota,
|
||||
* pas de spawn yt-dlp). Retourne un pseudo dump yt-dlp prêt pour pickTrack().
|
||||
*/
|
||||
export async function getCaptionTracksViaInnerTube(videoId) {
|
||||
const id = String(videoId || '').trim();
|
||||
if (!id) return { subtitles: {}, automatic_captions: {}, languages: [], trackCount: 0 };
|
||||
ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
|
||||
const yt = await getSession();
|
||||
try {
|
||||
const info = await yt.getInfo(id);
|
||||
const raw = info?.captions?.caption_tracks || [];
|
||||
return mapCaptionTracks(Array.isArray(raw) ? raw : Array.from(raw));
|
||||
} catch (e) {
|
||||
if (e?.code === 'yt_innertube_unavailable') throw e;
|
||||
throw Object.assign(new Error(`InnerTube captions failed: ${String(e?.message || e).slice(0, 160)}`), { ytStatus: 502, code: 'yt_innertube_failed' });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
// Recherche / channel / trending YouTube SANS clé API via yt-dlp (Step 17 P1).
|
||||
// Binaire résolu via YT_DLP_PATH > PATH. Jamais de secret en log.
|
||||
import { execFile as execFileCb } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { getYtDlpTimeoutMs, buildYtDlpExtraArgs, resolveYtDlpBin } from './youtube-common.mjs';
|
||||
|
||||
const execFileAsync = promisify(execFileCb);
|
||||
|
||||
/**
|
||||
* Normalise une entrée yt-dlp flat-playlist vers Suggestion (même contrat que youtube.mjs).
|
||||
* @param {any} e entrée yt-dlp
|
||||
*/
|
||||
export function mapFlatEntry(e) {
|
||||
if (!e || typeof e !== 'object') return null;
|
||||
const id = e.id || e.display_id || null;
|
||||
if (!id) return null;
|
||||
const thumbs = Array.isArray(e.thumbnails) ? e.thumbnails : [];
|
||||
const thumb = thumbs.length
|
||||
? (thumbs.find((t) => t?.url && (t.height || 0) >= 360)?.url || thumbs[thumbs.length - 1]?.url)
|
||||
: (e.thumbnail || undefined);
|
||||
const duration = e.duration != null ? Number(e.duration) : undefined;
|
||||
const views = e.view_count != null ? Number(e.view_count) : (e.views != null ? Number(e.views) : undefined);
|
||||
const channelId = e.channel_id || e.uploader_id || undefined;
|
||||
const uploaderName = e.channel || e.uploader || undefined;
|
||||
return {
|
||||
title: e.title || '',
|
||||
id: String(id),
|
||||
url: String(id).startsWith('http') ? String(id) : `https://www.youtube.com/watch?v=${id}`,
|
||||
thumbnail: thumb,
|
||||
uploaderName,
|
||||
type: 'video',
|
||||
...(Number.isFinite(duration) && duration > 0 ? { duration } : {}),
|
||||
...(Number.isFinite(views) ? { views } : {}),
|
||||
...(e.timestamp ? { publishedAt: new Date(Number(e.timestamp) * 1000).toISOString() } : {}),
|
||||
...(channelId ? { channelId: String(channelId), channelExternalId: String(channelId), channelUrl: `https://www.youtube.com/channel/${channelId}` } : {}),
|
||||
...(uploaderName ? { channelHandle: String(uploaderName) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Parse le JSON de `yt-dlp --dump-single-json --flat-playlist` (objet ou NDJSON fog). */
|
||||
export function parseFlatPlaylistJson(raw) {
|
||||
try {
|
||||
const text = String(raw || '').trim();
|
||||
if (!text) return [];
|
||||
// Cas standard : un seul objet JSON avec .entries
|
||||
try {
|
||||
const obj = JSON.parse(text);
|
||||
const entries = Array.isArray(obj?.entries) ? obj.entries : (Array.isArray(obj) ? obj : []);
|
||||
return entries.map(mapFlatEntry).filter(Boolean);
|
||||
} catch {
|
||||
// Fallback NDJSON (une ligne = un JSON)
|
||||
const out = [];
|
||||
for (const line of text.split('\n')) {
|
||||
const t = line.trim();
|
||||
if (!t) continue;
|
||||
try {
|
||||
const o = JSON.parse(t);
|
||||
const m = mapFlatEntry(o);
|
||||
if (m) out.push(m);
|
||||
} catch {}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
export function classifyScrapeError(e) {
|
||||
if (e?.code === 'yt_scrape_no_binary') return e;
|
||||
const msg = String(e?.message || e || '');
|
||||
if (/ENOENT/i.test(msg) || e?.code === 'ENOENT' || e?.errno === 'ENOENT') {
|
||||
return Object.assign(
|
||||
new Error('yt-dlp introuvable (ni PATH ni bundled). Installez yt-dlp ou définissez YT_DLP_PATH.'),
|
||||
{ ytStatus: 503, code: 'yt_scrape_no_binary' },
|
||||
);
|
||||
}
|
||||
if (/bot|sign in to confirm|challenge|cookies|login/i.test(msg)) {
|
||||
return Object.assign(new Error('YouTube bot-check (cookies/PO-Token requis)'), { ytStatus: 502, code: 'yt_scrape_bot_check' });
|
||||
}
|
||||
if (/timed out|timeout|ETIMEDOUT|killed|SIGKILL/i.test(msg)) {
|
||||
return Object.assign(new Error('YouTube scrape timeout'), { ytStatus: 504, code: 'yt_scrape_timeout' });
|
||||
}
|
||||
if (/unable to|unsupported|not available|private|deleted/i.test(msg)) {
|
||||
return Object.assign(new Error(`YouTube scrape upstream: ${msg.slice(0, 160)}`), { ytStatus: 502, code: 'yt_scrape_upstream' });
|
||||
}
|
||||
return Object.assign(new Error(`YouTube scrape failed: ${msg.slice(0, 200)}`), { ytStatus: 502, code: 'yt_scrape_failed' });
|
||||
}
|
||||
|
||||
async function runYtDlpFlat(queryOrUrl, { limit = 10, playlistStart = 1 } = {}) {
|
||||
const bin = await resolveYtDlpBin();
|
||||
const timeout = getYtDlpTimeoutMs();
|
||||
const end = playlistStart + Math.max(1, Math.min(50, Number(limit || 10))) - 1;
|
||||
const args = [
|
||||
'--dump-single-json',
|
||||
'--flat-playlist',
|
||||
'--no-warnings',
|
||||
'--no-check-certificates',
|
||||
'--skip-download',
|
||||
'--no-playlist',
|
||||
'--playlist-start', String(playlistStart),
|
||||
'--playlist-end', String(end),
|
||||
...buildYtDlpExtraArgs(),
|
||||
queryOrUrl,
|
||||
];
|
||||
try {
|
||||
const { stdout } = await execFileAsync(bin, args, { timeout, maxBuffer: 16 * 1024 * 1024 });
|
||||
return parseFlatPlaylistJson(stdout);
|
||||
} catch (e) {
|
||||
// yt-dlp peut écrire du JSON partiel sur stdout même en exit non-zero
|
||||
const partial = e?.stdout ? parseFlatPlaylistJson(String(e.stdout)) : [];
|
||||
if (partial.length) return partial;
|
||||
throw classifyScrapeError(e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Recherche sans clé. sort: relevance|date|views */
|
||||
export async function searchViaScrape(q, opts = {}) {
|
||||
const query = String(q || '').trim();
|
||||
if (query.length < 2) return [];
|
||||
const limit = Math.min(50, Math.max(1, Number(opts?.limit || 10)));
|
||||
const page = Math.max(1, Number(opts?.page || 1));
|
||||
const sort = String(opts?.sort || 'relevance').toLowerCase();
|
||||
const perPage = limit;
|
||||
const start = (page - 1) * perPage + 1;
|
||||
let prefix = 'ytsearch';
|
||||
if (sort === 'date') prefix = 'ytsearchdate';
|
||||
// views : pas de préfixe natif stable -> ytsearch + tri local
|
||||
const target = `${prefix}${perPage}:${query}`;
|
||||
let items = await runYtDlpFlat(target, { limit: perPage, playlistStart: start });
|
||||
if (sort === 'views') items = [...items].sort((a, b) => (b.views || 0) - (a.views || 0));
|
||||
return items.slice(0, perPage);
|
||||
}
|
||||
|
||||
const CHANNEL_TABS = { videos: 'videos', shorts: 'shorts', streams: 'streams', live: 'streams', playlists: 'playlists' };
|
||||
|
||||
/** Contenu chaîne sans clé. type: videos|shorts|playlists|live */
|
||||
export async function fetchChannelViaScrape(externalId, { type = 'videos', page = 1, limit = 24 } = {}) {
|
||||
const raw = String(externalId || '').trim().replace(/^@/, '');
|
||||
if (!raw) return { items: [], nextPage: null };
|
||||
const tab = CHANNEL_TABS[String(type)] || 'videos';
|
||||
const perPage = Math.min(50, Math.max(1, Number(limit || 24)));
|
||||
const pageNum = Math.max(1, Number(page || 1));
|
||||
const start = (pageNum - 1) * perPage + 1;
|
||||
// UC... -> /channel/UC.../tab, sinon -> /@/handle/tab
|
||||
const base = /^UC[\w-]{20,}$/.test(raw)
|
||||
? `https://www.youtube.com/channel/${raw}/${tab}`
|
||||
: `https://www.youtube.com/@${raw}/${tab}`;
|
||||
const items = await runYtDlpFlat(base, { limit: perPage, playlistStart: start });
|
||||
// playlists renvoient des ids PL... : on les garde tels quels avec un shape playlist
|
||||
if (tab === 'playlists') {
|
||||
const mapped = items.map((s) => ({ id: s.id, title: s.title, thumbnail: s.thumbnail, videoCount: null, updatedAt: s.publishedAt || null }));
|
||||
return { items: mapped, nextPage: items.length >= perPage ? pageNum + 1 : null };
|
||||
}
|
||||
let filtered = items;
|
||||
if (type === 'shorts') filtered = items.filter((s) => !s.duration || s.duration <= 70);
|
||||
return { items: filtered, nextPage: items.length >= perPage ? pageNum + 1 : null };
|
||||
}
|
||||
|
||||
/** Trending sans clé. Les onglets /feed/trending ont été retirés côté YouTube
|
||||
* (redirect home -> erreur tab) ; on tente les tabs puis repli ytsearch trié par vues. */
|
||||
export async function getTrendingViaScrape(limit = 24) {
|
||||
const n = Math.min(50, Math.max(1, Number(limit || 24)));
|
||||
const tabs = [
|
||||
'https://www.youtube.com/feed/trending',
|
||||
'https://www.youtube.com/trending',
|
||||
];
|
||||
for (const url of tabs) {
|
||||
try {
|
||||
const items = await runYtDlpFlat(url, { limit: n, playlistStart: 1 });
|
||||
if (items.length) return items;
|
||||
} catch {}
|
||||
}
|
||||
// Repli : recherche générique triée par vues (0 quota, toujours disponible)
|
||||
return runYtDlpFlat(`ytsearch${n}:top trending videos world`, { limit: n, playlistStart: 1 });
|
||||
}
|
||||
|
||||
/** Résout un channel_id via scrape (imprime channel_id sans appel API). */
|
||||
export async function resolveChannelIdViaScrape(externalId) {
|
||||
const raw = String(externalId || '').trim();
|
||||
if (/^UC[\w-]{20,}$/.test(raw)) return raw;
|
||||
const handle = raw.replace(/^@/, '');
|
||||
const bin = await resolveYtDlpBin().catch(() => null);
|
||||
if (!bin) {
|
||||
throw Object.assign(
|
||||
new Error('yt-dlp introuvable (ni PATH ni bundled). Installez yt-dlp ou définissez YT_DLP_PATH.'),
|
||||
{ ytStatus: 503, code: 'yt_scrape_no_binary' },
|
||||
);
|
||||
}
|
||||
const timeout = Math.min(15000, getYtDlpTimeoutMs());
|
||||
try {
|
||||
const { stdout } = await execFileAsync(bin,
|
||||
['--print', '%(channel_id)s', '--no-warnings', '--skip-download', '--playlist-end', '1', ...buildYtDlpExtraArgs(), `https://www.youtube.com/@${handle}/videos`],
|
||||
{ timeout });
|
||||
const id = String(stdout || '').trim().split('\n')[0].trim();
|
||||
if (/^UC[\w-]{20,}$/.test(id)) return id;
|
||||
} catch {}
|
||||
return raw;
|
||||
}
|
||||
+268
-143
@@ -9,6 +9,12 @@
|
||||
* @property {string=} uploaderName
|
||||
* @property {string=} type
|
||||
*/
|
||||
import {
|
||||
getYouTubeKeys, isKeyFailure, getSearchMode, getScrapeTtlMs,
|
||||
hashSearchKey, ytMetrics,
|
||||
} from './youtube-common.mjs';
|
||||
import { searchViaScrape } from './youtube-scrape.mjs';
|
||||
import { searchViaInnerTube } from './youtube-innertube.mjs';
|
||||
|
||||
function parseISODurationToSeconds(iso) {
|
||||
if (typeof iso !== 'string' || !iso) return 0;
|
||||
@@ -20,50 +26,6 @@ function parseISODurationToSeconds(iso) {
|
||||
return (hours * 3600) + (minutes * 60) + seconds;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve configured YouTube API keys.
|
||||
* Accepts YOUTUBE_API_KEYS as JSON array ('["k1","k2"]') or CSV ('k1,k2'),
|
||||
* plus the legacy single YOUTUBE_API_KEY as fallback. De-duplicated.
|
||||
* @returns {string[]}
|
||||
*/
|
||||
function getYouTubeKeys() {
|
||||
const keys = [];
|
||||
try {
|
||||
const raw = process.env.YOUTUBE_API_KEYS;
|
||||
if (raw && String(raw).trim() && String(raw).trim() !== 'undefined' && String(raw).trim() !== 'null') {
|
||||
const s = String(raw).trim();
|
||||
if (s.startsWith('[')) {
|
||||
try {
|
||||
const arr = JSON.parse(s);
|
||||
if (Array.isArray(arr)) keys.push(...arr.map(v => String(v || '').trim()).filter(Boolean));
|
||||
} catch {}
|
||||
} else {
|
||||
keys.push(...s.split(',').map(v => String(v || '').trim()).filter(Boolean));
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
try {
|
||||
const single = process.env.YOUTUBE_API_KEY;
|
||||
if (single && String(single).trim()) keys.push(String(single).trim());
|
||||
} catch {}
|
||||
return Array.from(new Set(keys.filter(Boolean)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a YouTube API failure as retryable with another key.
|
||||
* - 400 API_KEY_INVALID / "API key expired" -> key is dead, try next.
|
||||
* - 403 quotaExceeded / rateLimitExceeded / dailyLimitExceeded -> quota, try next.
|
||||
*/
|
||||
function isKeyFailure(status, data) {
|
||||
try {
|
||||
const reason = data?.error?.errors?.[0]?.reason || '';
|
||||
const message = String(data?.error?.message || '');
|
||||
if (status === 400 && (reason === 'API_KEY_INVALID' || /api key (expired|invalid)/i.test(message))) return true;
|
||||
if (status === 403 && /quota|rateLimit|dailyLimit|userRateLimit/i.test(reason + ' ' + message)) return true;
|
||||
} catch {}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** GET JSON from YouTube, rotating through keys on key failures. */
|
||||
async function ytFetchJson(base, paramsWithoutKey) {
|
||||
const keys = getYouTubeKeys();
|
||||
@@ -76,13 +38,19 @@ async function ytFetchJson(base, paramsWithoutKey) {
|
||||
for (const key of keys) {
|
||||
const params = new URLSearchParams(paramsWithoutKey);
|
||||
params.set('key', key);
|
||||
ytMetrics.apiCalls++;
|
||||
try {
|
||||
const { incYoutubeMetrics } = await import('../db.mjs').catch(() => ({}));
|
||||
if (String(base).includes('/search')) incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 100 });
|
||||
else incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 1 });
|
||||
ytMetrics.quotaUnits += String(base).includes('/search') ? 100 : 1;
|
||||
} catch {}
|
||||
const resp = await fetch(`${base}?${params.toString()}`);
|
||||
const data = await resp.json().catch(() => ({}));
|
||||
if (resp.ok) return data;
|
||||
lastStatus = resp.status;
|
||||
lastData = data;
|
||||
lastError = new Error(`YouTube API error: ${resp.status} ${data?.error?.message || ''}`.trim());
|
||||
// Only rotate to the next key on key-attributable failures; otherwise fail fast.
|
||||
if (!isKeyFailure(resp.status, data)) break;
|
||||
console.warn(`[YouTube] key ...${String(key).slice(-4)} failed (${resp.status}), trying next key`);
|
||||
}
|
||||
@@ -92,121 +60,278 @@ async function ytFetchJson(base, paramsWithoutKey) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a `suggestqueries.google.com` (client=youtube) response body into plain strings.
|
||||
* The endpoint answers either bare JSON (`["q",["s1","s2"],...]`) or wrapped in
|
||||
* `window.google.ac.h(...)`. Never throws: unparsable bodies yield [].
|
||||
* @param {string} text
|
||||
* @returns {string[]}
|
||||
*/
|
||||
export function parseYoutubeSuggestResponse(text) {
|
||||
try {
|
||||
const raw = String(text || '').trim();
|
||||
if (!raw) return [];
|
||||
let payload = raw;
|
||||
const start = raw.indexOf('(');
|
||||
const end = raw.lastIndexOf(')');
|
||||
if (start !== -1 && end > start) payload = raw.slice(start + 1, end);
|
||||
const parsed = JSON.parse(payload);
|
||||
const candidates = Array.isArray(parsed) && Array.isArray(parsed[1]) ? parsed[1] : [];
|
||||
return candidates
|
||||
.map((s) => {
|
||||
if (Array.isArray(s)) s = s[0];
|
||||
return (typeof s === 'string' ? s : String(s ?? '')).trim();
|
||||
})
|
||||
.filter(Boolean)
|
||||
.slice(0, 20);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Mémoire LRU process-wide pour le dispatcher (complète SQLite)
|
||||
const memCache = new Map();
|
||||
const MEM_MAX = Number(process.env.YT_SCRAPE_MEM_MAX || 300);
|
||||
function memGet(k, ttl) {
|
||||
const hit = memCache.get(k);
|
||||
if (!hit) return null;
|
||||
if ((Date.now() - hit.ts) >= ttl) { memCache.delete(k); return null; }
|
||||
memCache.delete(k); memCache.set(k, hit);
|
||||
return hit.items;
|
||||
}
|
||||
function memSet(k, items) {
|
||||
if (memCache.has(k)) memCache.delete(k);
|
||||
memCache.set(k, { ts: Date.now(), items });
|
||||
while (memCache.size > MEM_MAX) { const o = memCache.keys().next().value; if (o === undefined) break; memCache.delete(o); }
|
||||
}
|
||||
export function ytScrapeCacheStats() {
|
||||
return { memEntries: memCache.size, memMax: MEM_MAX };
|
||||
}
|
||||
|
||||
async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance' } = {}) {
|
||||
const keys = getYouTubeKeys();
|
||||
if (!keys.length) {
|
||||
throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' });
|
||||
}
|
||||
let order = 'relevance';
|
||||
if (sort === 'date') order = 'date';
|
||||
else if (sort === 'views') order = 'viewCount';
|
||||
const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
|
||||
const targetPage = Math.max(1, Number(page || 1));
|
||||
let pageToken = '';
|
||||
let currentPage = 1;
|
||||
let lastItems = [];
|
||||
while (currentPage <= targetPage) {
|
||||
const params = {
|
||||
part: 'snippet', q, type: 'video', maxResults: String(perPage), order,
|
||||
videoEmbeddable: 'true', safeSearch: 'moderate',
|
||||
};
|
||||
if (pageToken) params.pageToken = pageToken;
|
||||
const data = await ytFetchJson('https://www.googleapis.com/youtube/v3/search', params);
|
||||
if (currentPage === targetPage) { lastItems = Array.isArray(data.items) ? data.items : []; break; }
|
||||
const next = data.nextPageToken;
|
||||
if (!next) { lastItems = []; break; }
|
||||
pageToken = String(next);
|
||||
currentPage++;
|
||||
}
|
||||
const videoIds = (lastItems || []).map((item) => item?.id?.videoId).filter(Boolean);
|
||||
const detailsMap = new Map();
|
||||
if (videoIds.length > 0) {
|
||||
try {
|
||||
const detailsData = await ytFetchJson('https://www.googleapis.com/youtube/v3/videos', {
|
||||
part: 'contentDetails,statistics,status', id: videoIds.join(','),
|
||||
});
|
||||
for (const vid of detailsData?.items || []) if (vid?.id) detailsMap.set(vid.id, vid);
|
||||
} catch (e) {
|
||||
console.warn('[YouTube] details fetch failed, continuing without durations:', e?.message || e);
|
||||
}
|
||||
}
|
||||
return (lastItems || []).map((item) => {
|
||||
const videoId = item?.id?.videoId;
|
||||
const snippet = item?.snippet || {};
|
||||
const thumb = snippet.thumbnails?.high?.url || snippet.thumbnails?.medium?.url || snippet.thumbnails?.default?.url || undefined;
|
||||
const details = videoId ? detailsMap.get(videoId) : null;
|
||||
const duration = parseISODurationToSeconds(details?.contentDetails?.duration || '');
|
||||
const views = details?.statistics?.viewCount != null ? Number(details.statistics.viewCount) : undefined;
|
||||
const channelId = snippet.channelId || undefined;
|
||||
const embeddable = details?.status ? details.status.embeddable !== false : undefined;
|
||||
return {
|
||||
title: snippet.title || '', id: videoId,
|
||||
url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined,
|
||||
thumbnail: thumb, uploaderName: snippet.channelTitle || undefined, type: 'video',
|
||||
duration: duration > 0 ? duration : undefined, views,
|
||||
publishedAt: snippet.publishedAt || undefined, channelId,
|
||||
channelHandle: snippet.channelTitle || undefined, channelExternalId: channelId,
|
||||
channelUrl: channelId ? `https://www.youtube.com/channel/${channelId}` : undefined, embeddable,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** @type {{ id: 'yt', label: string, search: (q: string, opts: { limit: number, page?: number, sort?: 'relevance'|'date'|'views' }) => Promise<Suggestion[]> }} */
|
||||
const handler = {
|
||||
id: 'yt',
|
||||
label: 'YouTube',
|
||||
async search(q, opts) {
|
||||
const { limit = 10, page = 1, sort = 'relevance' } = opts || {};
|
||||
const mode = getSearchMode();
|
||||
const ttl = getScrapeTtlMs();
|
||||
const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
|
||||
const key = `yt|${hashSearchKey(`${String(q).toLowerCase().trim()}|${perPage}|${page}|${sort}|${mode}`)}`;
|
||||
// 1) mémoire
|
||||
const memHit = memGet(key, ttl);
|
||||
if (memHit) { ytMetrics.scrapeHits++; return memHit; }
|
||||
// 2) SQLite
|
||||
try {
|
||||
const keys = getYouTubeKeys();
|
||||
if (!keys.length) {
|
||||
throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' });
|
||||
const { getCachedYoutubeSearch } = await import('../db.mjs');
|
||||
const cached = getCachedYoutubeSearch(key);
|
||||
if (cached?.items) {
|
||||
ytMetrics.scrapeHits++;
|
||||
memSet(key, cached.items);
|
||||
return cached.items;
|
||||
}
|
||||
|
||||
let order = 'relevance';
|
||||
if (sort === 'date') order = 'date';
|
||||
else if (sort === 'views') order = 'viewCount';
|
||||
|
||||
// Iterate nextPageToken to reach the requested page (1-based)
|
||||
const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
|
||||
const targetPage = Math.max(1, Number(page || 1));
|
||||
let pageToken = '';
|
||||
let currentPage = 1;
|
||||
let lastItems = [];
|
||||
|
||||
while (currentPage <= targetPage) {
|
||||
const params = {
|
||||
part: 'snippet',
|
||||
q: q,
|
||||
type: 'video',
|
||||
maxResults: String(perPage),
|
||||
order,
|
||||
// Ne retourner que des vidéos lisibles en embed (évite l'erreur 153 côté player).
|
||||
// Les vidéos non-embeddables sont filtrées via le champ status ci-dessous pour /videos.
|
||||
videoEmbeddable: 'true',
|
||||
safeSearch: 'moderate',
|
||||
};
|
||||
if (pageToken) params.pageToken = pageToken;
|
||||
|
||||
const data = await ytFetchJson('https://www.googleapis.com/youtube/v3/search', params);
|
||||
|
||||
if (currentPage === targetPage) {
|
||||
lastItems = Array.isArray(data.items) ? data.items : [];
|
||||
break;
|
||||
}
|
||||
|
||||
// Prepare for next iteration
|
||||
const next = data.nextPageToken;
|
||||
if (!next) {
|
||||
// No more pages; requested page beyond available results
|
||||
lastItems = [];
|
||||
break;
|
||||
}
|
||||
pageToken = String(next);
|
||||
currentPage++;
|
||||
} catch {}
|
||||
const persist = async (items, source) => {
|
||||
// Ne jamais persister un résultat vide : une page vide transitoire
|
||||
// (continuation expirée, raté réseau partiel) ne doit pas empoisonner
|
||||
// le cache et bloquer les requêtes suivantes.
|
||||
if (!Array.isArray(items) || items.length === 0) return items;
|
||||
memSet(key, items);
|
||||
try {
|
||||
const { setCachedYoutubeSearch, incYoutubeMetrics } = await import('../db.mjs');
|
||||
setCachedYoutubeSearch(key, q, items, source, ttl);
|
||||
if (source === 'scrape') incYoutubeMetrics({ scrapeCalls: 1 });
|
||||
} catch {}
|
||||
return items;
|
||||
};
|
||||
const t0 = Date.now();
|
||||
const tryScrape = async () => {
|
||||
ytMetrics.scrapeCalls++;
|
||||
try { const { incYoutubeMetrics } = await import('../db.mjs'); incYoutubeMetrics({ scrapeCalls: 1 }); } catch {}
|
||||
return searchViaScrape(q, { limit: perPage, page, sort });
|
||||
};
|
||||
const tryApi = () => searchViaApi(q, { limit: perPage, page, sort });
|
||||
const tryInnerTube = () => searchViaInnerTube(q, { limit: perPage, page, sort });
|
||||
const log = (source, extra = '') => console.log(`[YT search] source=${source} mode=${mode} latency=${Date.now() - t0}ms results=${extra}`);
|
||||
try {
|
||||
if (mode === 'api-only') {
|
||||
const items = await tryApi();
|
||||
log('api', items.length);
|
||||
return persist(items, 'api');
|
||||
}
|
||||
|
||||
const videoIds = (lastItems || [])
|
||||
.map(item => item?.id?.videoId)
|
||||
.filter(Boolean);
|
||||
|
||||
const detailsMap = new Map();
|
||||
if (videoIds.length > 0) {
|
||||
if (mode === 'scrape-only') {
|
||||
const items = await tryScrape();
|
||||
log('scrape', items.length);
|
||||
return persist(items, 'scrape');
|
||||
}
|
||||
if (mode === 'innertube-only') {
|
||||
const items = await tryInnerTube();
|
||||
log('innertube', items.length);
|
||||
return persist(items, 'innertube');
|
||||
}
|
||||
if (mode === 'api-first') {
|
||||
try {
|
||||
const detailsData = await ytFetchJson('https://www.googleapis.com/youtube/v3/videos', {
|
||||
part: 'contentDetails,statistics,status',
|
||||
id: videoIds.join(','),
|
||||
});
|
||||
for (const vid of detailsData?.items || []) {
|
||||
if (vid?.id) detailsMap.set(vid.id, vid);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[YouTube] details fetch failed, continuing without durations:', e?.message || e);
|
||||
const items = await tryApi();
|
||||
log('api', items.length);
|
||||
return persist(items, 'api');
|
||||
} catch (apiErr) {
|
||||
ytMetrics.fallbacks++;
|
||||
console.warn(`[YT search] api-first fallback to scrape: ${apiErr?.message || apiErr}`);
|
||||
const items = await tryScrape();
|
||||
log('scrape(fallback)', items.length);
|
||||
return persist(items, 'scrape');
|
||||
}
|
||||
}
|
||||
|
||||
return (lastItems || []).map(item => {
|
||||
const videoId = item?.id?.videoId;
|
||||
const snippet = item?.snippet || {};
|
||||
const thumb = snippet.thumbnails?.high?.url
|
||||
|| snippet.thumbnails?.medium?.url
|
||||
|| snippet.thumbnails?.default?.url
|
||||
|| undefined;
|
||||
|
||||
const details = videoId ? detailsMap.get(videoId) : null;
|
||||
const isoDuration = details?.contentDetails?.duration || '';
|
||||
const duration = parseISODurationToSeconds(isoDuration);
|
||||
const views = details?.statistics?.viewCount != null ? Number(details.statistics.viewCount) : undefined;
|
||||
const channelId = snippet.channelId || undefined;
|
||||
const channelHandle = snippet.channelTitle || undefined;
|
||||
// status.embeddable === false -> le player renvoie l'erreur 153 ("Video configuration error").
|
||||
const embeddable = details?.status ? details.status.embeddable !== false : undefined;
|
||||
|
||||
return {
|
||||
title: snippet.title || '',
|
||||
id: videoId,
|
||||
url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined,
|
||||
thumbnail: thumb,
|
||||
uploaderName: snippet.channelTitle || undefined,
|
||||
type: 'video',
|
||||
duration: duration > 0 ? duration : undefined,
|
||||
views,
|
||||
publishedAt: snippet.publishedAt || undefined,
|
||||
channelId,
|
||||
channelHandle,
|
||||
channelExternalId: channelId,
|
||||
channelUrl: channelId ? `https://www.youtube.com/channel/${channelId}` : undefined,
|
||||
embeddable,
|
||||
};
|
||||
});
|
||||
if (mode === 'scrape-first') {
|
||||
// scrape -> API (comportement historique, sans InnerTube)
|
||||
try {
|
||||
const items = await tryScrape();
|
||||
log('scrape', items.length);
|
||||
return persist(items, 'scrape');
|
||||
} catch (scrapeErr) {
|
||||
ytMetrics.scrapeErrors++;
|
||||
ytMetrics.fallbacks++;
|
||||
console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`);
|
||||
try {
|
||||
const items = await tryApi();
|
||||
log('api(fallback)', items.length);
|
||||
return persist(items, 'api');
|
||||
} catch (apiErr) {
|
||||
const noBinary = scrapeErr?.code === 'yt_scrape_no_binary';
|
||||
const noKey = apiErr?.code === 'youtube_api_key_unavailable';
|
||||
if (noBinary && noKey) {
|
||||
const err = new Error('YouTube indisponible : yt-dlp introuvable (définissez YT_DLP_PATH) et aucune clé YOUTUBE_API_KEY configurée.');
|
||||
err.ytStatus = 503; err.code = 'youtube_no_source';
|
||||
throw err;
|
||||
}
|
||||
console.error(`[YT search] both sources failed. scrape=${scrapeErr?.message} api=${apiErr?.message}`);
|
||||
throw scrapeErr;
|
||||
}
|
||||
}
|
||||
}
|
||||
// innertube-first (défaut) : InnerTube -> scrape yt-dlp -> API officielle.
|
||||
// Chaque couche ne fait jamais échouer la recherche à elle seule.
|
||||
const errors = [];
|
||||
try {
|
||||
const items = await tryInnerTube();
|
||||
log('innertube', items.length);
|
||||
return persist(items, 'innertube');
|
||||
} catch (itErr) {
|
||||
ytMetrics.innertubeErrors = (ytMetrics.innertubeErrors || 0) + 1;
|
||||
ytMetrics.fallbacks++;
|
||||
errors.push(`innertube=${itErr?.message}`);
|
||||
console.warn(`[YT search] innertube failed (${itErr?.code || 'unknown'}), fallback to scrape`);
|
||||
}
|
||||
try {
|
||||
const items = await tryScrape();
|
||||
log('scrape(fallback)', items.length);
|
||||
return persist(items, 'scrape');
|
||||
} catch (scrapeErr) {
|
||||
ytMetrics.scrapeErrors++;
|
||||
ytMetrics.fallbacks++;
|
||||
errors.push(`scrape=${scrapeErr?.message}`);
|
||||
console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`);
|
||||
}
|
||||
try {
|
||||
const items = await tryApi();
|
||||
log('api(fallback)', items.length);
|
||||
return persist(items, 'api');
|
||||
} catch (apiErr) {
|
||||
errors.push(`api=${apiErr?.message}`);
|
||||
const noSource = !getYouTubeKeys().length;
|
||||
if (noSource) {
|
||||
const err = new Error(`YouTube indisponible (${errors.join(' | ').slice(0, 220)}). Vérifiez le réseau/YT_DLP_PATH ou ajoutez YOUTUBE_API_KEYS.`);
|
||||
err.ytStatus = 503; err.code = 'youtube_no_source';
|
||||
throw err;
|
||||
}
|
||||
console.error(`[YT search] all sources failed. ${errors.join(' | ')}`);
|
||||
const err = new Error(errors[0] || 'search_failed');
|
||||
err.ytStatus = 502; err.code = 'youtube_all_sources_failed';
|
||||
throw err;
|
||||
}
|
||||
} catch (error) {
|
||||
// Log explicite (clé expirée / quota) puis remonte l'erreur pour que
|
||||
// /api/search la reporte dans `errors.yt` au lieu d'un groupe vide silencieux.
|
||||
const status = error?.ytStatus || 500;
|
||||
console.error(`YouTube search error (status ${status}):`, error?.message || error);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
async suggest(q, opts) {
|
||||
const limit = Math.min(20, Math.max(1, Number(opts?.limit || 10)));
|
||||
const query = String(q || '').trim();
|
||||
if (query.length < 2) return [];
|
||||
try {
|
||||
const params = new URLSearchParams({ client: 'youtube', ds: 'yt', hl: 'fr', q: query });
|
||||
const headers = { 'User-Agent': 'Mozilla/5.0' };
|
||||
const proxy = String(process.env.YT_EGRESS_PROXY || '').trim();
|
||||
const fetchOpts = { headers, signal: AbortSignal.timeout(6000) };
|
||||
// Note : suggest reste sans proxy (endpoint Google public léger) sauf si egress configuré côté infra.
|
||||
if (proxy) console.debug?.('[YT suggest] egress proxy configured, direct fetch kept for suggest');
|
||||
const resp = await fetch(`https://suggestqueries.google.com/complete/search?${params.toString()}`, fetchOpts);
|
||||
if (!resp.ok) return [];
|
||||
const text = await resp.text();
|
||||
return parseYoutubeSuggestResponse(text).slice(0, limit);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
// Step 15 — Suggest tests: unit parsing/dedup (offline) + API contract (isolated server).
|
||||
// Run with: npm run test:suggest
|
||||
// Scenario coverage (todo Step 15):
|
||||
// - providers=yt returns only yt suggestions
|
||||
// - unknown provider falls back to the full registry
|
||||
// Assertions are on fan-out shape, never on external provider content.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { spawn } from 'node:child_process';
|
||||
import net from 'node:net';
|
||||
|
||||
function expect(cond, msg) {
|
||||
if (!cond) throw new Error(`Assertion failed: ${msg}`);
|
||||
}
|
||||
function logOk(msg) { console.log(`✓ ${msg}`); }
|
||||
|
||||
// ---------- 1) Offline unit tests: parsers + graceful degradation ----------
|
||||
const yt = (await import('../providers/youtube.mjs')).default;
|
||||
const { parseYoutubeSuggestResponse } = await import('../providers/youtube.mjs');
|
||||
const dm = (await import('../providers/dailymotion.mjs')).default;
|
||||
|
||||
// Bare JSON payload
|
||||
expect(
|
||||
JSON.stringify(parseYoutubeSuggestResponse('["tut",["tutoriel angular","tutoriel android"],[],{}]')) ===
|
||||
JSON.stringify(['tutoriel angular', 'tutoriel android']),
|
||||
'bare JSON suggest payload parses',
|
||||
);
|
||||
logOk('parseYoutubeSuggestResponse bare JSON');
|
||||
|
||||
// window.google.ac.h(...) wrapped payload
|
||||
expect(
|
||||
JSON.stringify(
|
||||
parseYoutubeSuggestResponse('window.google.ac.h(["tut",["tutoriel angular"],[],{}])'),
|
||||
) === JSON.stringify(['tutoriel angular']),
|
||||
'wrapped JSONP suggest payload parses',
|
||||
);
|
||||
logOk('parseYoutubeSuggestResponse wrapped payload');
|
||||
|
||||
// Array-wrapped suggestions (newer format: [text, type, ...])
|
||||
expect(
|
||||
JSON.stringify(parseYoutubeSuggestResponse('["tut",[["tutoriel angular",0],["tutoriel android",0]]]')) ===
|
||||
JSON.stringify(['tutoriel angular', 'tutoriel android']),
|
||||
'array-wrapped suggestions keep text only',
|
||||
);
|
||||
logOk('parseYoutubeSuggestResponse array-wrapped entries');
|
||||
|
||||
// Garbage / empty payloads never throw
|
||||
expect(parseYoutubeSuggestResponse('not json at all').length === 0, 'garbage payload yields []');
|
||||
expect(parseYoutubeSuggestResponse('').length === 0, 'empty payload yields []');
|
||||
logOk('parseYoutubeSuggestResponse graceful degradation');
|
||||
|
||||
// min-length guard: no network below 2 chars (would throw offline otherwise)
|
||||
expect(JSON.stringify(await yt.suggest('a')) === '[]', 'yt.suggest short query returns [] without network');
|
||||
expect(JSON.stringify(await dm.suggest('x')) === '[]', 'dm.suggest short query returns [] without network');
|
||||
logOk('suggest min-length guard (no network)');
|
||||
|
||||
// ---------- 2) API contract against a real isolated server ----------
|
||||
const PORT = await new Promise((resolve) => {
|
||||
const srv = net.createServer();
|
||||
srv.listen(0, '127.0.0.1', () => {
|
||||
const p = srv.address().port;
|
||||
srv.close(() => resolve(p));
|
||||
});
|
||||
});
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'newtube-suggest-test-'));
|
||||
const server = spawn(process.execPath, ['./server/index.mjs'], {
|
||||
env: { ...process.env, PORT: String(PORT), NEWTUBE_DB_FILE: path.join(tmpDir, 'suggest.db'), JWT_SECRET: 'suggest-test-secret', NODE_ENV: 'test' },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
cwd: path.resolve(import.meta.dirname, '..', '..'),
|
||||
});
|
||||
let serverLogs = '';
|
||||
server.stdout.on('data', (d) => { serverLogs += d.toString(); });
|
||||
server.stderr.on('data', (d) => { serverLogs += d.toString(); });
|
||||
const baseUrl = `http://127.0.0.1:${PORT}`;
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
async function waitForServer(timeoutMs = 20000) {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
try {
|
||||
const res = await fetch(`${baseUrl}/`);
|
||||
if (res.status < 500) return true;
|
||||
} catch {}
|
||||
await sleep(300);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
let failures = 0;
|
||||
async function scenario(name, fn) {
|
||||
try { await fn(); logOk(name); }
|
||||
catch (e) { failures += 1; console.error(`✗ ${name}`); console.error(e?.stack || String(e)); }
|
||||
}
|
||||
|
||||
try {
|
||||
const up = await waitForServer();
|
||||
expect(up, 'isolated server starts');
|
||||
|
||||
await scenario('q too short -> 400', async () => {
|
||||
const res = await fetch(`${baseUrl}/api/search/suggest?q=a`);
|
||||
expect(res.status === 400, `expected 400, got ${res.status}`);
|
||||
});
|
||||
|
||||
await scenario('providers=yt returns only yt suggestions', async () => {
|
||||
const res = await fetch(`${baseUrl}/api/search/suggest?q=tutoriel&providers=yt&limit=5`);
|
||||
expect(res.status === 200, `expected 200, got ${res.status}`);
|
||||
const body = await res.json();
|
||||
expect(body.q === 'tutoriel', 'q echoed');
|
||||
expect(JSON.stringify(Object.keys(body.groups)) === JSON.stringify(['yt']), `only yt group (got ${Object.keys(body.groups)})`);
|
||||
expect(Array.isArray(body.groups.yt), 'yt group is an array');
|
||||
expect(body.groups.yt.length <= 5, 'limit respected');
|
||||
});
|
||||
|
||||
await scenario('unknown provider falls back to the full registry', async () => {
|
||||
const res = await fetch(`${baseUrl}/api/search/suggest?q=tutoriel&providers=xx,yy&limit=3`);
|
||||
expect(res.status === 200, `expected 200, got ${res.status}`);
|
||||
const body = await res.json();
|
||||
const keys = Object.keys(body.groups).sort();
|
||||
expect(JSON.stringify(keys) === JSON.stringify(['dm', 'od', 'pt', 'ru', 'tw', 'yt']), `full registry fallback (got ${keys})`);
|
||||
});
|
||||
} finally {
|
||||
server.kill();
|
||||
}
|
||||
|
||||
if (failures > 0) {
|
||||
console.error(`\n${failures} suggest scenario(s) failed. Server logs:\n${serverLogs}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('\nAll suggest tests passed.');
|
||||
@@ -0,0 +1,212 @@
|
||||
// Step 16 — Transcript tests (offline fixtures for parsers + API contract).
|
||||
// Run with: npm run test:transcript
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { spawn } from 'node:child_process';
|
||||
import net from 'node:net';
|
||||
import {
|
||||
pickTrack,
|
||||
parseJson3,
|
||||
parseVtt,
|
||||
parseTrackText,
|
||||
parseXmlCaptions,
|
||||
orderedTracks,
|
||||
looksLikeHtmlError,
|
||||
capLines,
|
||||
normalizeTranscriptProvider,
|
||||
} from '../transcript.mjs';
|
||||
|
||||
const track = (url, ext = 'vtt') => ({ url, ext });
|
||||
|
||||
describe('pickTrack', () => {
|
||||
it('prefers manual subtitles over automatic captions', () => {
|
||||
const json = {
|
||||
subtitles: { en: [track('https://x/en.vtt')] },
|
||||
automatic_captions: { en: [track('https://x/en.auto.vtt')] },
|
||||
};
|
||||
const { track: t, lang } = pickTrack(json, 'en');
|
||||
assert.equal(t.url, 'https://x/en.vtt');
|
||||
assert.equal(lang, 'en');
|
||||
});
|
||||
|
||||
it('falls back fr -> fr-* -> auto -> first available', () => {
|
||||
const json = {
|
||||
subtitles: { 'fr-CA': [track('https://x/frca.vtt')] },
|
||||
automatic_captions: { en: [track('https://x/en.auto.vtt')] },
|
||||
};
|
||||
assert.equal(pickTrack(json, 'fr').lang, 'fr-CA');
|
||||
// 'de' matches nothing: first available (manual preferred)
|
||||
const fb = pickTrack(json, 'de');
|
||||
assert.equal(fb.lang, 'fr-CA');
|
||||
// auto-only dict
|
||||
const autoOnly = pickTrack({ automatic_captions: { en: [track('https://x/e.vtt')] } }, 'en');
|
||||
assert.equal(autoOnly.track.url, 'https://x/e.vtt');
|
||||
});
|
||||
|
||||
it('returns null track when no subtitles exist', () => {
|
||||
assert.deepEqual(pickTrack({}, 'fr'), { track: null, languages: [], lang: null });
|
||||
assert.deepEqual(pickTrack({ subtitles: {}, automatic_captions: {} }, 'en'), {
|
||||
track: null,
|
||||
languages: [],
|
||||
lang: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('exposes the language list for the UI selector', () => {
|
||||
const { languages } = pickTrack(
|
||||
{ subtitles: { fr: [track('a')], en: [track('b')] }, automatic_captions: { es: [track('c')] } },
|
||||
'fr',
|
||||
);
|
||||
assert.deepEqual([...languages].sort(), ['en', 'es', 'fr']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseJson3', () => {
|
||||
it('converts events with segs to normalized lines', () => {
|
||||
const lines = parseJson3({
|
||||
events: [
|
||||
{ tStartMs: 0, dDurationMs: 2500, segs: [{ utf8: 'Bonjour' }] },
|
||||
{ tStartMs: 2500, dDurationMs: 3100, segs: [{ utf8: 'Bien' }, { utf8: 'venue' }] },
|
||||
],
|
||||
});
|
||||
assert.deepEqual(lines, [
|
||||
{ t: 0, dur: 2.5, text: 'Bonjour' },
|
||||
{ t: 2.5, dur: 3.1, text: 'Bienvenue' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('drops events without segs or with empty text', () => {
|
||||
const lines = parseJson3({ events: [{ tStartMs: 0 }, { tStartMs: 1, dDurationMs: 1, segs: [{ utf8: ' ' }] }] });
|
||||
assert.deepEqual(lines, []);
|
||||
assert.deepEqual(parseJson3({}), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseVtt', () => {
|
||||
const vtt = `WEBVTT
|
||||
|
||||
00:00:00.000 --> 00:00:02.500
|
||||
Bonjour <c.colorE5E5E5>à tous</c>
|
||||
|
||||
12
|
||||
00:00:02.500 --> 00:00:05.600
|
||||
Bienvenue
|
||||
dans cette vidéo
|
||||
`;
|
||||
it('parses cues with identifiers, tags and multi-line content', () => {
|
||||
const lines = parseVtt(vtt);
|
||||
assert.equal(lines.length, 2);
|
||||
assert.equal(lines[0].t, 0);
|
||||
assert.equal(lines[0].dur, 2.5);
|
||||
assert.equal(lines[0].text, 'Bonjour à tous');
|
||||
assert.equal(lines[1].text, 'Bienvenue dans cette vidéo');
|
||||
assert.ok(Math.abs(lines[1].t - 2.5) < 1e-9);
|
||||
});
|
||||
|
||||
it('supports MM:SS.mmm timestamps and decodes entities', () => {
|
||||
const lines = parseVtt('WEBVTT\n\n01:02.000 --> 01:04.500\nFish & chips\n');
|
||||
assert.equal(lines.length, 1);
|
||||
assert.equal(lines[0].t, 62);
|
||||
assert.equal(lines[0].text, 'Fish & chips');
|
||||
});
|
||||
|
||||
it('ignores garbage blocks', () => {
|
||||
assert.deepEqual(parseVtt('WEBVTT\n\nNOTE nothing here\n'), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe('orderedTracks + parseTrackText + XML captions', () => {
|
||||
const auto = {
|
||||
fr: [{ url: 'https://x/fr.json3', ext: 'json3' }],
|
||||
en: [{ url: 'https://x/en.json3', ext: 'json3' }],
|
||||
es: [{ url: 'https://x/es.vtt', ext: 'vtt' }],
|
||||
};
|
||||
it('tries requested lang first, then original en', () => {
|
||||
const langs = orderedTracks({ subtitles: {}, automatic_captions: auto }, 'fr').map((o) => o.lang);
|
||||
assert.deepEqual(langs, ['fr', 'en', 'es']);
|
||||
});
|
||||
it('dedupes identical track URLs', () => {
|
||||
const dup = { subtitles: {}, automatic_captions: { en: [{ url: 'https://x/same', ext: 'vtt' }], fr: [{ url: 'https://x/same', ext: 'vtt' }] } };
|
||||
assert.equal(orderedTracks(dup, 'fr').length, 1);
|
||||
});
|
||||
it('detects HTML error pages instead of JSON-parsing them', () => {
|
||||
assert.equal(looksLikeHtmlError('<!DOCTYPE html><html>Sorry</html>'), true);
|
||||
assert.equal(looksLikeHtmlError('WEBVTT\n\n00:00:00.000 --> 1'), false);
|
||||
assert.deepEqual(parseTrackText('<!DOCTYPE html> nope', 'json3'), []);
|
||||
});
|
||||
it('parses srv/ttml XML captions', () => {
|
||||
const srv = parseXmlCaptions('<transcript><p t="0" d="2500">Hello</p></transcript>');
|
||||
assert.equal(srv.length, 1);
|
||||
assert.equal(srv[0].text, 'Hello');
|
||||
const ttml = parseXmlCaptions('<tt><body><div><p begin="00:00:01.000" end="00:00:03.500">Bonjour</p></div></body></tt>');
|
||||
assert.equal(ttml.length, 1);
|
||||
assert.equal(ttml[0].t, 1);
|
||||
});
|
||||
it('parseTrackText handles json3 and vtt payloads', () => {
|
||||
const j = parseTrackText(JSON.stringify({ events: [{ tStartMs: 0, dDurationMs: 1000, segs: [{ utf8: 'Hi' }] }] }), 'json3');
|
||||
assert.equal(j.length, 1);
|
||||
const v = parseTrackText('WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nHello\n', 'vtt');
|
||||
assert.equal(v.length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('capLines + normalizeTranscriptProvider', () => { it('truncates very long transcripts', () => {
|
||||
const lines = Array.from({ length: 10 }, (_, i) => ({ t: i, dur: 1, text: `l${i}` }));
|
||||
assert.equal(capLines(lines, 3).length, 3);
|
||||
});
|
||||
it('maps short and long provider ids', () => {
|
||||
assert.equal(normalizeTranscriptProvider('yt'), 'youtube');
|
||||
assert.equal(normalizeTranscriptProvider('youtube'), 'youtube');
|
||||
assert.equal(normalizeTranscriptProvider('dm'), 'dailymotion');
|
||||
assert.equal(normalizeTranscriptProvider('pt'), 'peertube');
|
||||
assert.equal(normalizeTranscriptProvider('xx'), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('API contract', () => {
|
||||
it('rejects unknown providers with 400 { available: false }', async () => {
|
||||
const port = await new Promise((resolve) => {
|
||||
const srv = net.createServer();
|
||||
srv.listen(0, '127.0.0.1', () => {
|
||||
const p = srv.address().port;
|
||||
srv.close(() => resolve(p));
|
||||
});
|
||||
});
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'newtube-transcript-test-'));
|
||||
const server = spawn(process.execPath, ['./server/index.mjs'], {
|
||||
env: {
|
||||
...process.env,
|
||||
PORT: String(port),
|
||||
NEWTUBE_DB_FILE: path.join(tmpDir, 'transcript.db'),
|
||||
JWT_SECRET: 'transcript-test-secret',
|
||||
NODE_ENV: 'test',
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
cwd: path.resolve(import.meta.dirname, '..', '..'),
|
||||
});
|
||||
const baseUrl = `http://127.0.0.1:${port}`;
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
try {
|
||||
let up = false;
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < 20000) {
|
||||
try {
|
||||
const res = await fetch(`${baseUrl}/`);
|
||||
if (res.status < 500) { up = true; break; }
|
||||
} catch {}
|
||||
await sleep(300);
|
||||
}
|
||||
assert.ok(up, 'isolated server starts');
|
||||
const res = await fetch(`${baseUrl}/api/transcript/xx/abc123?lang=fr`);
|
||||
assert.equal(res.status, 400);
|
||||
const body = await res.json();
|
||||
assert.equal(body.available, false);
|
||||
assert.ok(typeof body.error === 'string');
|
||||
} finally {
|
||||
server.kill();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
// Step 18 — tests offline InnerTube (AUCUN réseau : mappers purs + modes).
|
||||
import assert from 'node:assert/strict';
|
||||
import { mapVideoNode, mapLockupView, mapNodes, mapCaptionTracks, parseViewsText, parseDurationLabel } from '../providers/youtube-innertube.mjs';
|
||||
import { pickTrack, orderedTracks } from '../transcript.mjs';
|
||||
import { getSearchMode, YT_SEARCH_MODES } from '../providers/youtube-common.mjs';
|
||||
|
||||
function ok(msg) { console.log(`✓ ${msg}`); }
|
||||
|
||||
// 1) parseViewsText FR/EN
|
||||
{
|
||||
assert.equal(parseViewsText('81 973 vues'), 81973);
|
||||
assert.equal(parseViewsText('1,2 M vues'), 1200000);
|
||||
assert.equal(parseViewsText('57K'), 57000);
|
||||
assert.equal(parseViewsText('3 k vues'), 3000);
|
||||
assert.equal(parseViewsText(''), undefined);
|
||||
assert.equal(parseViewsText(null), undefined);
|
||||
ok('parseViewsText FR/EN');
|
||||
}
|
||||
|
||||
// 1b) parseDurationLabel (a11y)
|
||||
{
|
||||
assert.equal(parseDurationLabel("What's new in Angular 18 2 minutes, 27 seconds"), 147);
|
||||
assert.equal(parseDurationLabel('Live 1 heure, 5 minutes'), 3900);
|
||||
assert.equal(parseDurationLabel('nonsense'), undefined);
|
||||
ok('parseDurationLabel');
|
||||
}
|
||||
|
||||
// 2) mapVideoNode type Video (forme réelle youtubei.js v18)
|
||||
{
|
||||
const m = mapVideoNode({
|
||||
type: 'Video', id: 'abc123',
|
||||
title: { text: 'Demo' },
|
||||
duration: { text: '15:22', seconds: 922 },
|
||||
thumbnails: [{ url: 'http://t/low.jpg' }, { url: 'http://t/high.jpg' }],
|
||||
author: { id: 'UC999', name: 'Chaine' },
|
||||
published: { text: 'il y a 2 ans' },
|
||||
view_count: { text: '81 973 vues' },
|
||||
badges: [{ label: 'Sous-titres' }],
|
||||
is_live: false,
|
||||
});
|
||||
assert.equal(m.id, 'abc123');
|
||||
assert.equal(m.url, 'https://www.youtube.com/watch?v=abc123');
|
||||
assert.equal(m.thumbnail, 'http://t/high.jpg');
|
||||
assert.equal(m.duration, 922);
|
||||
assert.equal(m.views, 81973);
|
||||
assert.equal(m.channelId, 'UC999');
|
||||
assert.deepEqual(m.badges, ['Sous-titres']);
|
||||
assert.equal(m.isShort, undefined);
|
||||
ok('mapVideoNode Video');
|
||||
}
|
||||
|
||||
// 3) Shorts/Reel -> isShort ; live -> isLive ; bruit filtré
|
||||
{
|
||||
const s = mapVideoNode({ type: 'ShortsLockupView', id: 'sh1', title: { text: 'Short' }, thumbnails: [{ url: 'u' }] });
|
||||
assert.equal(s.isShort, true);
|
||||
const l = mapVideoNode({ type: 'Video', id: 'lv1', title: { text: 'Live' }, thumbnails: [{ url: 'u' }], is_live: true });
|
||||
assert.equal(l.isLive, true);
|
||||
assert.equal(mapVideoNode({ type: 'Playlist', id: 'PL1', title: { text: 'PL' } }), null);
|
||||
assert.equal(mapVideoNode({ type: 'Channel', id: 'UC1', title: { text: 'C' } }), null);
|
||||
assert.equal(mapVideoNode(null), null);
|
||||
assert.equal(mapVideoNode({ type: 'Video', title: { text: 'no id' } }), null);
|
||||
ok('mapVideoNode shorts/live/filtrage');
|
||||
}
|
||||
|
||||
// 4) mapNodes déduplique
|
||||
{
|
||||
const mk = (id) => ({ type: 'Video', id, title: { text: id }, thumbnails: [{ url: 'u' }] });
|
||||
const out = mapNodes([mk('a'), mk('a'), mk('b'), null]);
|
||||
assert.deepEqual(out.map((x) => x.id), ['a', 'b']);
|
||||
ok('mapNodes dédup');
|
||||
}
|
||||
|
||||
// 4b) mapLockupView (renderer moderne du watch-next façon SmartTube)
|
||||
{
|
||||
const m = mapLockupView({
|
||||
type: 'LockupView', content_id: '0wIL6d5TxGc', content_type: 'VIDEO',
|
||||
content_image: { image: [{ url: 'http://t/small.jpg', width: 168 }, { url: 'http://t/big.jpg', width: 336 }] },
|
||||
metadata: {
|
||||
title: { text: "What's new" },
|
||||
metadata: { metadata_rows: [
|
||||
{ metadata_parts: [{ text: { text: 'Rainer Hahnekamp' } }] },
|
||||
{ metadata_parts: [{ text: { text: '57K' } }] },
|
||||
] },
|
||||
image: { a11y_label: 'Go to channel Rainer Hahnekamp' },
|
||||
},
|
||||
renderer_context: { accessibility_context: { label: "What's new 2 minutes, 27 seconds" } },
|
||||
});
|
||||
assert.equal(m.id, '0wIL6d5TxGc');
|
||||
assert.equal(m.url, 'https://www.youtube.com/watch?v=0wIL6d5TxGc');
|
||||
assert.equal(m.thumbnail, 'http://t/big.jpg');
|
||||
assert.equal(m.uploaderName, 'Rainer Hahnekamp');
|
||||
assert.equal(m.views, 57000);
|
||||
assert.equal(m.duration, 147);
|
||||
assert.equal(mapVideoNode({ type: 'LockupView', content_id: 'x', content_type: 'VIDEO', metadata: { title: { text: 'T' } } }).id, 'x');
|
||||
assert.equal(mapLockupView({ type: 'LockupView', content_type: 'PLAYLIST', content_id: 'PL1', metadata: { title: { text: 'PL' } } }), null);
|
||||
ok('mapLockupView watch-next');
|
||||
}
|
||||
|
||||
// 5) modes innertube connus
|
||||
{
|
||||
assert.ok(YT_SEARCH_MODES.includes('innertube-first'));
|
||||
assert.ok(YT_SEARCH_MODES.includes('innertube-only'));
|
||||
const prev = process.env.YT_SEARCH_MODE;
|
||||
process.env.YT_SEARCH_MODE = 'innertube-only';
|
||||
assert.equal(getSearchMode(), 'innertube-only');
|
||||
if (prev === undefined) delete process.env.YT_SEARCH_MODE; else process.env.YT_SEARCH_MODE = prev;
|
||||
ok('modes innertube-first/only');
|
||||
}
|
||||
|
||||
// 6) mapCaptionTracks : forme InnerTube -> pseudo dump yt-dlp, interop pickTrack
|
||||
{
|
||||
const cap = mapCaptionTracks([
|
||||
{ base_url: 'https://x/t?caps=asr&x=1', language_code: 'en', kind: 'asr', name: { text: 'English (auto-generated)' } },
|
||||
{ base_url: 'https://x/t?caps=m&x=2', language_code: 'fr', name: { text: 'Français' } },
|
||||
{ base_url: '', language_code: 'de', name: 'Deutsch' },
|
||||
]);
|
||||
assert.deepEqual(cap.languages.sort(), ['en', 'fr']);
|
||||
assert.equal(cap.trackCount, 2);
|
||||
assert.match(cap.automatic_captions.en[0].url, /fmt=json3/);
|
||||
assert.equal(cap.automatic_captions.en[0].ext, 'json3');
|
||||
assert.equal(cap.subtitles.fr[0].name, 'Français');
|
||||
// Interop : le pseudo dump passe dans pickTrack/orderedTracks sans modification
|
||||
const picked = pickTrack({ subtitles: cap.subtitles, automatic_captions: cap.automatic_captions }, 'fr');
|
||||
assert.equal(picked.lang, 'fr');
|
||||
assert.ok(picked.track.url.includes('fmt=json3'));
|
||||
const ordered = orderedTracks({ subtitles: cap.subtitles, automatic_captions: cap.automatic_captions }, 'de');
|
||||
assert.ok(ordered.length >= 2);
|
||||
ok('mapCaptionTracks + interop pickTrack/orderedTracks');
|
||||
}
|
||||
|
||||
console.log('youtube-innertube tests passed');
|
||||
@@ -0,0 +1,98 @@
|
||||
// Step 17 — tests offline : parsers scrape + dispatcher (AUCUN réseau, AUCUNE clé).
|
||||
import assert from 'node:assert/strict';
|
||||
import { mapFlatEntry, parseFlatPlaylistJson, classifyScrapeError } from '../providers/youtube-scrape.mjs';
|
||||
import { getSearchMode, hashSearchKey, buildYtDlpExtraArgs, resolveYtDlpBin, resetYtDlpBinCache } from '../providers/youtube-common.mjs';
|
||||
|
||||
function ok(msg) { console.log(`✓ ${msg}`); }
|
||||
|
||||
// 1) mapFlatEntry
|
||||
{
|
||||
const s = mapFlatEntry({
|
||||
id: 'dQw4w9WgXcQ', title: 'Demo', channel_id: 'UC123', channel: 'Chaine',
|
||||
duration: 212, view_count: 1000, timestamp: 1700000000,
|
||||
thumbnails: [{ url: 'http://t/low.jpg', height: 90 }, { url: 'http://t/high.jpg', height: 720 }],
|
||||
});
|
||||
assert.equal(s.id, 'dQw4w9WgXcQ');
|
||||
assert.equal(s.url, 'https://www.youtube.com/watch?v=dQw4w9WgXcQ');
|
||||
assert.equal(s.thumbnail, 'http://t/high.jpg');
|
||||
assert.equal(s.duration, 212);
|
||||
ok('mapFlatEntry maps flat entry to Suggestion');
|
||||
}
|
||||
assert.equal(mapFlatEntry(null), null);
|
||||
assert.equal(mapFlatEntry({}), null);
|
||||
ok('mapFlatEntry null-safe');
|
||||
|
||||
// 2) parseFlatPlaylistJson : objet + NDJSON + vide
|
||||
{
|
||||
const raw = JSON.stringify({ entries: [{ id: 'a', title: 'A' }, { id: '', title: 'x' }, null] });
|
||||
assert.equal(parseFlatPlaylistJson(raw).length, 1);
|
||||
ok('parseFlatPlaylistJson objet .entries');
|
||||
}
|
||||
{
|
||||
const raw = `{"id":"a","title":"A"}\n{"id":"b","title":"B"}\nnot-json\n`;
|
||||
assert.equal(parseFlatPlaylistJson(raw).length, 2);
|
||||
ok('parseFlatPlaylistJson NDJSON tolerant');
|
||||
}
|
||||
assert.deepEqual(parseFlatPlaylistJson(''), []);
|
||||
assert.deepEqual(parseFlatPlaylistJson('{{{'), []);
|
||||
ok('parseFlatPlaylistJson vide/invalide -> []');
|
||||
|
||||
// 3) getSearchMode défaut innertube-first + normalisation
|
||||
{
|
||||
const prev = process.env.YT_SEARCH_MODE;
|
||||
delete process.env.YT_SEARCH_MODE;
|
||||
assert.equal(getSearchMode(), 'innertube-first');
|
||||
process.env.YT_SEARCH_MODE = 'API-FIRST';
|
||||
assert.equal(getSearchMode(), 'api-first');
|
||||
process.env.YT_SEARCH_MODE = 'scrape-first';
|
||||
assert.equal(getSearchMode(), 'scrape-first');
|
||||
process.env.YT_SEARCH_MODE = 'nonsense';
|
||||
assert.equal(getSearchMode(), 'innertube-first');
|
||||
if (prev === undefined) delete process.env.YT_SEARCH_MODE; else process.env.YT_SEARCH_MODE = prev;
|
||||
ok('getSearchMode défaut + normalisation');
|
||||
}
|
||||
|
||||
// 4) hash stable + extra args jamais de secret en clair autre que proxy path
|
||||
{
|
||||
assert.equal(hashSearchKey('a'), hashSearchKey('a'));
|
||||
assert.notEqual(hashSearchKey('a'), hashSearchKey('b'));
|
||||
const args = buildYtDlpExtraArgs();
|
||||
assert.ok(Array.isArray(args));
|
||||
ok('hashSearchKey + buildYtDlpExtraArgs');
|
||||
}
|
||||
|
||||
// 5) dispatcher scrape-only sans clé : ne doit PAS exiger YOUTUBE_API_KEY au import,
|
||||
// et searchViaScrape vide sur query courte
|
||||
{
|
||||
const { searchViaScrape } = await import('../providers/youtube-scrape.mjs');
|
||||
assert.deepEqual(await searchViaScrape('a'), []);
|
||||
ok('searchViaScrape query courte -> [] sans réseau');
|
||||
}
|
||||
|
||||
// 6) classifyScrapeError : ENOENT -> yt_scrape_no_binary (message actionnable, jamais de spawn brut en UI)
|
||||
{
|
||||
const e = classifyScrapeError(Object.assign(new Error('spawn yt-dlp ENOENT'), { code: 'ENOENT', errno: 'ENOENT' }));
|
||||
assert.equal(e.code, 'yt_scrape_no_binary');
|
||||
assert.equal(e.ytStatus, 503);
|
||||
assert.match(e.message, /YT_DLP_PATH/);
|
||||
const passthrough = classifyScrapeError(Object.assign(new Error('x'), { code: 'yt_scrape_no_binary' }));
|
||||
assert.equal(passthrough.code, 'yt_scrape_no_binary');
|
||||
ok('classifyScrapeError ENOENT -> yt_scrape_no_binary');
|
||||
}
|
||||
|
||||
// 7) resolveYtDlpBin : YT_DLP_PATH inexistant -> fallback PATH/bundled (jamais de throw si un binaire existe)
|
||||
{
|
||||
resetYtDlpBinCache();
|
||||
const prev = process.env.YT_DLP_PATH;
|
||||
process.env.YT_DLP_PATH = '/nonexistent/yt-dlp';
|
||||
try {
|
||||
const bin = await resolveYtDlpBin();
|
||||
assert.ok(typeof bin === 'string' && bin.length > 0);
|
||||
ok(`resolveYtDlpBin fallback -> ${bin}`);
|
||||
} finally {
|
||||
resetYtDlpBinCache();
|
||||
if (prev === undefined) delete process.env.YT_DLP_PATH; else process.env.YT_DLP_PATH = prev;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('youtube-scrape tests passed');
|
||||
@@ -0,0 +1,309 @@
|
||||
// Step 16 — Pure transcript helpers (no network, no DB).
|
||||
// Data source: `yt-dlp --dump-single-json --skip-download` exposes
|
||||
// `subtitles` (manual) and `automatic_captions` (auto-generated).
|
||||
// Track entries are arrays of { url, ext, name } (or single objects).
|
||||
|
||||
/**
|
||||
* @typedef {{ t: number, dur: number, text: string }} TranscriptLine
|
||||
*/
|
||||
|
||||
const MAX_LINES_DEFAULT = Number(process.env.TRANSCRIPT_MAX_LINES || 5000);
|
||||
|
||||
/** Normalize a language code for comparison (lowercase, `_` -> `-`). */
|
||||
function normLang(code) {
|
||||
return String(code || '').trim().toLowerCase().replace(/_/g, '-');
|
||||
}
|
||||
|
||||
/** Pick the first usable track object from an array-or-single entry. */
|
||||
function firstTrack(entry) {
|
||||
if (!entry) return null;
|
||||
const list = Array.isArray(entry) ? entry : [entry];
|
||||
return list.find((t) => t && typeof t.url === 'string' && t.url) || null;
|
||||
}
|
||||
|
||||
function trackExtOf(track) {
|
||||
const ext = String(track?.ext || '').toLowerCase();
|
||||
if (ext) return ext;
|
||||
try {
|
||||
const u = new URL(String(track?.url || ''));
|
||||
const m = /\.([a-z0-9]+)(?:[?#]|$)/i.exec(u.pathname || '');
|
||||
if (m) return m[1].toLowerCase();
|
||||
} catch {}
|
||||
return '';
|
||||
}
|
||||
|
||||
/** Find a language key in `dict` matching `code` exactly or by prefix (`fr` -> `fr-*`). */
|
||||
function findLangKey(dict, code) {
|
||||
const want = normLang(code);
|
||||
if (!want || !dict) return null;
|
||||
const keys = Object.keys(dict);
|
||||
const exact = keys.find((k) => normLang(k) === want);
|
||||
if (exact) return exact;
|
||||
// `fr-ca` requested, `fr` available (and vice versa): match on primary subtag
|
||||
const primary = want.split('-')[0];
|
||||
return keys.find((k) => normLang(k).split('-')[0] === primary) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Select the best subtitle track.
|
||||
* Priority: manual exact > manual prefix/primary > auto exact > auto prefix/primary > first available.
|
||||
* @param {any} json yt-dlp dump-single-json
|
||||
* @param {string} [lang]
|
||||
* @returns {{ track: any|null, languages: string[], lang: string|null }}
|
||||
*/
|
||||
export function pickTrack(json, lang = 'fr') {
|
||||
const manual = (json && json.subtitles) || {};
|
||||
const auto = (json && json.automatic_captions) || {};
|
||||
const manualKeys = Object.keys(manual);
|
||||
const autoKeys = Object.keys(auto);
|
||||
const languages = Array.from(new Set([...manualKeys, ...autoKeys]));
|
||||
|
||||
if (languages.length === 0) return { track: null, languages: [], lang: null };
|
||||
|
||||
const manualKey = findLangKey(manual, lang);
|
||||
if (manualKey) {
|
||||
const track = firstTrack(manual[manualKey]);
|
||||
if (track) return { track, languages, lang: manualKey };
|
||||
}
|
||||
const autoKey = findLangKey(auto, lang);
|
||||
if (autoKey) {
|
||||
const track = firstTrack(auto[autoKey]);
|
||||
if (track) return { track, languages, lang: autoKey };
|
||||
}
|
||||
// Fallback: first available track (manual preferred)
|
||||
for (const key of manualKeys) {
|
||||
const track = firstTrack(manual[key]);
|
||||
if (track) return { track, languages, lang: key };
|
||||
}
|
||||
for (const key of autoKeys) {
|
||||
const track = firstTrack(auto[key]);
|
||||
if (track) return { track, languages, lang: key };
|
||||
}
|
||||
return { track: null, languages, lang: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Order candidate tracks to try when the preferred one cannot be fetched.
|
||||
* YouTube auto-translated tracks (`tlang=xx`) are often rate-limited (HTTP 429
|
||||
* or "Sorry" HTML pages) while the original language still works: always try
|
||||
* the requested language first, then the original (`en`), then everything
|
||||
* else (manual preferred). De-duplicated by URL.
|
||||
* @param {any} json yt-dlp dump-single-json
|
||||
* @param {string} [lang]
|
||||
* @returns {Array<{ track: any, lang: string|null }>}
|
||||
*/
|
||||
export function orderedTracks(json, lang = 'fr') {
|
||||
const manual = (json && json.subtitles) || {};
|
||||
const auto = (json && json.automatic_captions) || {};
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
const push = (key, dict) => {
|
||||
if (key == null) return;
|
||||
const t = firstTrack(dict[key]);
|
||||
if (!t || !t.url || seen.has(String(t.url))) return;
|
||||
seen.add(String(t.url));
|
||||
out.push({ track: t, lang: key });
|
||||
};
|
||||
const primary = pickTrack(json, lang);
|
||||
if (primary.track) {
|
||||
seen.add(String(primary.track.url));
|
||||
out.push({ track: primary.track, lang: primary.lang });
|
||||
}
|
||||
// Original language first fallback (avoids translated-track rate limits)
|
||||
const reqPrimary = String(lang || '').toLowerCase().split('-')[0];
|
||||
for (const fallbackLang of ['en', 'fr']) {
|
||||
if (fallbackLang === reqPrimary) continue;
|
||||
const k = findLangKey(manual, fallbackLang);
|
||||
if (k) push(k, manual);
|
||||
const ka = findLangKey(auto, fallbackLang);
|
||||
if (ka) push(ka, auto);
|
||||
}
|
||||
for (const key of Object.keys(manual)) push(key, manual);
|
||||
for (const key of Object.keys(auto)) push(key, auto);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a YouTube `json3` timedtext payload into normalized lines.
|
||||
* @param {any} data parsed JSON
|
||||
* @returns {TranscriptLine[]}
|
||||
*/
|
||||
export function parseJson3(data) {
|
||||
const events = (data && data.events) || [];
|
||||
if (!Array.isArray(events)) return [];
|
||||
const out = [];
|
||||
for (const e of events) {
|
||||
if (!e || !Array.isArray(e.segs)) continue;
|
||||
const text = e.segs.map((s) => String(s?.utf8 ?? '')).join('').replace(/\n+/g, ' ').trim();
|
||||
if (!text) continue;
|
||||
out.push({
|
||||
t: Number(e.tStartMs || 0) / 1000,
|
||||
dur: Number(e.dDurationMs || 0) / 1000,
|
||||
text,
|
||||
});
|
||||
}
|
||||
return capLines(out);
|
||||
}
|
||||
|
||||
/** Decode a handful of HTML entities found in VTT payloads. */
|
||||
function decodeEntities(s) {
|
||||
return String(s || '')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/ /g, ' ');
|
||||
}
|
||||
|
||||
/** Strip XML/HTML tags and decode entities to plain cue text. */
|
||||
function xmlTextToPlain(s) {
|
||||
return decodeEntities(String(s || '').replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim()).trim();
|
||||
}
|
||||
|
||||
function parseTimeAttrToSeconds(v) {
|
||||
if (v == null || v === '') return null;
|
||||
const s = String(v).trim();
|
||||
if (!s) return null;
|
||||
if (/^\d+(\.\d+)?$/.test(s)) return Number(s);
|
||||
const m = /(?:(\d+):)?([0-5]?\d):([0-5]\d)(?:\.(\d{1,3}))?/.exec(s);
|
||||
if (!m) return null;
|
||||
return Number(m[1] || 0) * 3600 + Number(m[2]) * 60 + Number(m[3]) + Number((m[4] || '0').padEnd(3, '0')) / 1000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse YouTube XML caption formats (`srv1`/`srv2`/`srv3` `<p>` cues,
|
||||
* `ttml` `<p begin= end=>` cues) into normalized lines.
|
||||
* @param {string} text raw XML
|
||||
* @returns {TranscriptLine[]}
|
||||
*/
|
||||
export function parseXmlCaptions(text) {
|
||||
const out = [];
|
||||
const src = String(text || '');
|
||||
if (!src || !/<(p|text|tt|transcript)\b/i.test(src)) return capLines(out);
|
||||
// YouTube srv*: <p t="0" d="2500">Hello <s>world</s></p>
|
||||
const srvRe = /<p\b[^>]*?(?:t|start)="([^"]*)"[^>]*?(?:d|dur)="([^"]*)"[^>]*>([\s\S]*?)<\/p\s*>/gi;
|
||||
// Generic ttml: <p begin="00:00:00.000" end="00:00:02.500">...</p>
|
||||
const ttmlRe = /<p\b[^>]*?begin="([^"]*)"[^>]*?end="([^"]*)"[^>]*>([\s\S]*?)<\/p\s*>/gi;
|
||||
let m;
|
||||
while ((m = srvRe.exec(src)) !== null) {
|
||||
const start = Number(m[1]) / 1000;
|
||||
const dur = Number(m[2]) / 1000;
|
||||
const cleaned = xmlTextToPlain(m[3]);
|
||||
if (!cleaned || !Number.isFinite(start)) continue;
|
||||
out.push({ t: start, dur: Number.isFinite(dur) && dur > 0 ? dur : 0, text: cleaned });
|
||||
}
|
||||
if (out.length === 0) {
|
||||
while ((m = ttmlRe.exec(src)) !== null) {
|
||||
const start = parseTimeAttrToSeconds(m[1]);
|
||||
const end = parseTimeAttrToSeconds(m[2]);
|
||||
const cleaned = xmlTextToPlain(m[3]);
|
||||
if (!cleaned || start == null || end == null) continue;
|
||||
out.push({ t: start, dur: Math.max(0, end - start), text: cleaned });
|
||||
}
|
||||
}
|
||||
// Plain <text start="2.5" dur="2.0">Fallback</text> variant
|
||||
if (out.length === 0) {
|
||||
const textRe = /<text\b[^>]*?start="([^"]*)"[^>]*?(?:dur="([^"]*)")?[^>]*>([\s\S]*?)<\/text\s*>/gi;
|
||||
while ((m = textRe.exec(src)) !== null) {
|
||||
const start = Number(m[1]);
|
||||
const dur = Number(m[2] || 0);
|
||||
const cleaned = xmlTextToPlain(m[3]);
|
||||
if (!cleaned || !Number.isFinite(start)) continue;
|
||||
out.push({ t: start, dur: dur > 0 ? dur : 0, text: cleaned });
|
||||
}
|
||||
}
|
||||
return capLines(out);
|
||||
}
|
||||
|
||||
/** Detect YouTube "Sorry / automated queries" style HTML error pages. */
|
||||
export function looksLikeHtmlError(text) {
|
||||
const s = String(text || '').trimStart().slice(0, 512).toLowerCase();
|
||||
return s.startsWith('<!doctype') || s.startsWith('<html');
|
||||
}
|
||||
|
||||
function vttTimestampToSeconds(h, m, s, ms) {
|
||||
return Number(h) * 3600 + Number(m) * 60 + Number(s) + Number(ms) / 1000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a WebVTT payload into normalized lines.
|
||||
* Handles cue identifiers, multi-line cues and inline tags (`<c>`, `<i>`, timestamps).
|
||||
* @param {string} text raw VTT
|
||||
* @returns {TranscriptLine[]}
|
||||
*/
|
||||
export function parseVtt(text) {
|
||||
if (looksLikeHtmlError(text)) return [];
|
||||
const out = [];
|
||||
const blocks = String(text || '').replace(/\r\n/g, '\n').split(/\n{2,}/);
|
||||
const tsRe = /(?:(\d{2,}):)?([0-5]?\d):([0-5]\d)\.(\d{3})\s*-->\s*(?:(\d{2,}):)?([0-5]?\d):([0-5]\d)\.(\d{3})/;
|
||||
for (const block of blocks) {
|
||||
const lines = String(block || '').split('\n');
|
||||
const timingIdx = lines.findIndex((l) => l.includes('-->'));
|
||||
if (timingIdx === -1) continue;
|
||||
const m = tsRe.exec(lines[timingIdx]);
|
||||
if (!m) continue;
|
||||
const start = vttTimestampToSeconds(m[1] || '0', m[2], m[3], m[4]);
|
||||
const end = vttTimestampToSeconds(m[5] || '0', m[6], m[7], m[8]);
|
||||
const content = lines
|
||||
.slice(timingIdx + 1)
|
||||
.join(' ')
|
||||
// strip inline VTT tags (<c.colorE5E5E5>, <i>, <00:00:01.000>, ...)
|
||||
.replace(/<[^>]*>/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
const decoded = decodeEntities(content).trim();
|
||||
// Skip WEBVTT header leftovers and empty cues
|
||||
if (!decoded || /^WEBVTT/i.test(decoded)) continue;
|
||||
out.push({ t: start, dur: Math.max(0, end - start), text: decoded });
|
||||
}
|
||||
return capLines(out);
|
||||
}
|
||||
|
||||
/** Bound response volume for very long transcripts (Phase 1 decision: truncate). */
|
||||
export function capLines(lines, max = MAX_LINES_DEFAULT) {
|
||||
const list = Array.isArray(lines) ? lines : [];
|
||||
const n = Math.max(1, Number(max || MAX_LINES_DEFAULT));
|
||||
return list.length > n ? list.slice(0, n) : list;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a downloaded timedtext payload regardless of its format.
|
||||
* Tries `json3`, then `vtt`, then XML captions (`srv*`/`ttml`).
|
||||
* HTML error pages always yield `[]`.
|
||||
*/
|
||||
export function parseTrackText(text, ext = '') {
|
||||
if (looksLikeHtmlError(text)) return [];
|
||||
const e = String(ext || '').toLowerCase();
|
||||
const asJson3 = () => {
|
||||
try { return parseJson3(JSON.parse(String(text))); } catch { return []; }
|
||||
};
|
||||
if (e === 'json3') {
|
||||
const lines = asJson3();
|
||||
if (lines.length) return lines;
|
||||
const vtt = parseVtt(text);
|
||||
if (vtt.length) return vtt;
|
||||
return parseXmlCaptions(text);
|
||||
}
|
||||
const vtt = parseVtt(text);
|
||||
if (vtt.length) return vtt;
|
||||
const j = asJson3();
|
||||
if (j.length) return j;
|
||||
return parseXmlCaptions(text);
|
||||
}
|
||||
|
||||
/** Map short (`yt`) and long (`youtube`) provider ids to `providerUrlFrom()` names. */
|
||||
export function normalizeTranscriptProvider(provider) {
|
||||
const p = String(provider || '').trim().toLowerCase();
|
||||
const map = {
|
||||
yt: 'youtube', youtube: 'youtube',
|
||||
dm: 'dailymotion', dailymotion: 'dailymotion',
|
||||
tw: 'twitch', twitch: 'twitch',
|
||||
pt: 'peertube', peertube: 'peertube',
|
||||
od: 'odysee', odysee: 'odysee',
|
||||
ru: 'rumble', rumble: 'rumble',
|
||||
};
|
||||
return map[p] || null;
|
||||
}
|
||||
|
||||
export { trackExtOf as transcriptTrackExt, firstTrack as transcriptFirstTrack };
|
||||
Reference in New Issue
Block a user