feat(youtube): InnerTube-first search, transcripts and watch-next related (Steps 15-18)
- InnerTube layer via pinned youtubei.js 18.1.0 (no quota, no key): search with merged continuations (unlimited pages), watch-next related with LockupView mapping, caption-track discovery - 3-layer dispatcher (YT_SEARCH_MODE, default innertube-first): innertube -> yt-dlp scrape -> official API, graceful errors.yt - Robust yt-dlp binary resolution (YT_DLP_PATH > PATH > bundled) with systematic API fallback (fixes spawn ENOENT in UI) - Transcript: InnerTube caption discovery (YT_TRANSCRIPT_SOURCE), reusing pickTrack/orderedTracks/parseTrackText; yt-dlp fallback kept - Watch: sidebar uses real watch-next related[] (/api/details), title-search fallback for other providers - Cache: memory LRU + SQLite (youtube_search_cache, youtube_metrics), never persist empty pages; /healthz observability; /api/trending - Includes pending Step 15/16 leftovers in same files (suggest, test scripts); unrelated provider adapters left uncommitted
This commit is contained in:
@@ -63,6 +63,16 @@ async function resolveYouTubeChannelId(externalId) {
|
||||
const raw = String(externalId || '').trim();
|
||||
if (/^UC[\w-]{20,}$/.test(raw)) return raw;
|
||||
const handle = raw.replace(/^@/, '');
|
||||
// 0) scrape sans clé (Step 17) : ne consomme aucun quota
|
||||
try {
|
||||
const { getSearchMode } = await import('./youtube-common.mjs');
|
||||
const mode = getSearchMode();
|
||||
if (mode !== 'api-only') {
|
||||
const { resolveChannelIdViaScrape } = await import('./youtube-scrape.mjs');
|
||||
const id = await resolveChannelIdViaScrape(raw);
|
||||
if (id && /^UC[\w-]{20,}$/.test(id)) return id;
|
||||
}
|
||||
} catch {}
|
||||
// 1) channels?forHandle (fonctionne encore pour beaucoup de chaînes)
|
||||
try {
|
||||
const data = await ytGet('channels', { part: 'id', forHandle: handle });
|
||||
@@ -131,9 +141,32 @@ function ytTokenFor(key, page) {
|
||||
}
|
||||
|
||||
async function ytContent(externalId, { type, page, limit, sort, q }) {
|
||||
const channelId = await resolveYouTubeChannelId(externalId);
|
||||
const perPage = Math.min(Math.max(1, Number(limit || 24)), 50);
|
||||
const pageNum = Math.max(1, Number(page || 1));
|
||||
// Step 17 : scrape-first sans clé (0 quota). Fallback API si bot-check/timeout.
|
||||
try {
|
||||
const { getSearchMode } = await import('./youtube-common.mjs');
|
||||
const mode = getSearchMode();
|
||||
if (mode !== 'api-only') {
|
||||
const { fetchChannelViaScrape } = await import('./youtube-scrape.mjs');
|
||||
// externalId brut (handle ou UC...) : le scrape gère les deux formes
|
||||
const scraped = await fetchChannelViaScrape(externalId, { type, page: pageNum, limit: perPage });
|
||||
if (Array.isArray(scraped?.items) && scraped.items.length) {
|
||||
if (mode === 'scrape-only') return { ...scraped, total: null };
|
||||
// scrape-first : retour direct si non vide
|
||||
return { ...scraped, total: null };
|
||||
}
|
||||
// vide -> on tente l'API (chaîne à faible volume ou tab non supporté en scrape)
|
||||
if (mode === 'scrape-only') return { items: [], nextPage: null };
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[channel-content/yt] scrape failed, fallback api:', e?.code || e?.message || e);
|
||||
try {
|
||||
const { getSearchMode } = await import('./youtube-common.mjs');
|
||||
if (getSearchMode() === 'scrape-only') return { items: [], nextPage: null };
|
||||
} catch {}
|
||||
}
|
||||
const channelId = await resolveYouTubeChannelId(externalId);
|
||||
if (type === 'playlists') {
|
||||
const key = ['pl', channelId, perPage].join('|');
|
||||
const token = ytTokenFor(key, pageNum);
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
// Helpers partagés YouTube : clés API, mode de recherche, anti-ban yt-dlp, métriques.
|
||||
// Extrait de youtube.mjs + index.mjs pour éviter la duplication (Step 17 P0).
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { execFile as execFileCb } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
export function getYouTubeKeys() {
|
||||
const keys = [];
|
||||
try {
|
||||
const raw = process.env.YOUTUBE_API_KEYS;
|
||||
if (raw && String(raw).trim() && !['undefined', 'null'].includes(String(raw).trim())) {
|
||||
const s = String(raw).trim();
|
||||
if (s.startsWith('[')) {
|
||||
try {
|
||||
const arr = JSON.parse(s);
|
||||
if (Array.isArray(arr)) keys.push(...arr.map((v) => String(v || '').trim()).filter(Boolean));
|
||||
} catch {}
|
||||
} else {
|
||||
keys.push(...s.split(',').map((v) => String(v || '').trim()).filter(Boolean));
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
try {
|
||||
const single = process.env.YOUTUBE_API_KEY;
|
||||
if (single && String(single).trim()) keys.push(String(single).trim());
|
||||
} catch {}
|
||||
return Array.from(new Set(keys.filter(Boolean)));
|
||||
}
|
||||
|
||||
export function isKeyFailure(status, data) {
|
||||
try {
|
||||
const reason = data?.error?.errors?.[0]?.reason || '';
|
||||
const message = String(data?.error?.message || '');
|
||||
if (status === 400 && (reason === 'API_KEY_INVALID' || /api key (expired|invalid)/i.test(message))) return true;
|
||||
if (status === 403 && /quota|rateLimit|dailyLimit|userRateLimit/i.test(`${reason} ${message}`)) return true;
|
||||
} catch {}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Mode de recherche YT : innertube-first (défaut) | innertube-only | scrape-first | api-first | scrape-only | api-only */
|
||||
export const YT_SEARCH_MODES = ['innertube-first', 'innertube-only', 'scrape-first', 'api-first', 'scrape-only', 'api-only'];
|
||||
export function getSearchMode() {
|
||||
const m = String(process.env.YT_SEARCH_MODE || 'innertube-first').trim().toLowerCase();
|
||||
if (YT_SEARCH_MODES.includes(m)) return m;
|
||||
return 'innertube-first';
|
||||
}
|
||||
|
||||
export function getScrapeTtlMs() {
|
||||
const v = Number(process.env.YT_SCRAPE_TTL_MS || 30 * 60 * 1000);
|
||||
return Number.isFinite(v) && v > 0 ? v : 30 * 60 * 1000;
|
||||
}
|
||||
|
||||
export function getYtDlpTimeoutMs() {
|
||||
const v = Number(process.env.YT_DLP_TIMEOUT_MS || 20000);
|
||||
return Number.isFinite(v) && v >= 5000 ? v : 20000;
|
||||
}
|
||||
|
||||
export function getYtDlpBin() {
|
||||
const p = String(process.env.YT_DLP_PATH || '').trim();
|
||||
if (p) return p;
|
||||
return process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp';
|
||||
}
|
||||
|
||||
/**
|
||||
* Résolution robuste du binaire yt-dlp (même ordre que server/index.mjs) :
|
||||
* 1. YT_DLP_PATH (si le fichier existe)
|
||||
* 2. `yt-dlp[.exe]` sur le PATH (probé via --version)
|
||||
* 3. binaire bundled de `youtube-dl-exec` (node_modules/.../bin/, utilisé en Docker)
|
||||
* Résultat mis en cache process-wide. Throw avec code `yt_scrape_no_binary` si introuvable.
|
||||
*/
|
||||
let _resolvedBin = null;
|
||||
export function resetYtDlpBinCache() { _resolvedBin = null; }
|
||||
|
||||
function bundledYtDlpPath() {
|
||||
try {
|
||||
const exe = process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp';
|
||||
// server/providers/ -> server/../node_modules/youtube-dl-exec/bin/
|
||||
const cand = path.resolve(path.dirname(new URL(import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, '$1')), '..', '..', 'node_modules', 'youtube-dl-exec', 'bin', exe);
|
||||
if (fs.existsSync(cand)) return cand;
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function getBundledYtDlpPath() { return bundledYtDlpPath(); }
|
||||
|
||||
async function probeBin(bin) {
|
||||
try {
|
||||
const execFileAsync = promisify(execFileCb);
|
||||
await execFileAsync(bin, ['--version'], { timeout: 10000 });
|
||||
return true;
|
||||
} catch (e) {
|
||||
// ENOENT = binaire absent ; autre erreur (timeout...) = présent mais KO -> on le garde quand même
|
||||
return e?.code !== 'ENOENT' && e?.errno !== 'ENOENT';
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveYtDlpBin() {
|
||||
if (_resolvedBin) return _resolvedBin;
|
||||
// 1) YT_DLP_PATH explicite
|
||||
try {
|
||||
const p = String(process.env.YT_DLP_PATH || '').trim();
|
||||
if (p && fs.existsSync(p)) {
|
||||
if (await probeBin(p)) { _resolvedBin = p; return p; }
|
||||
} else if (p) {
|
||||
console.warn(`[YT] YT_DLP_PATH introuvable : ${p} (fallback PATH/bundled)`);
|
||||
}
|
||||
} catch {}
|
||||
// 2) PATH
|
||||
const exe = process.platform === 'win32' ? 'yt-dlp.exe' : 'yt-dlp';
|
||||
for (const cand of [exe, 'yt-dlp']) {
|
||||
try {
|
||||
if (await probeBin(cand)) { _resolvedBin = cand; return cand; }
|
||||
} catch {}
|
||||
}
|
||||
// 3) bundled youtube-dl-exec
|
||||
const bundled = bundledYtDlpPath();
|
||||
if (bundled && await probeBin(bundled)) { _resolvedBin = bundled; return bundled; }
|
||||
throw Object.assign(
|
||||
new Error('yt-dlp introuvable (PATH, YT_DLP_PATH ni binaire bundled). Installez yt-dlp ou définissez YT_DLP_PATH.'),
|
||||
{ ytStatus: 503, code: 'yt_scrape_no_binary' },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Args anti-ban communs pour yt-dlp (cookies, PO-Token, proxy).
|
||||
* Ne jamais logger les valeurs (cookies path exclu des logs verbeux).
|
||||
*/
|
||||
export function buildYtDlpExtraArgs() {
|
||||
const args = [];
|
||||
try {
|
||||
const cookies = String(process.env.YT_COOKIES_FILE || '').trim();
|
||||
if (cookies && fs.existsSync(cookies)) {
|
||||
args.push('--cookies', cookies);
|
||||
}
|
||||
} catch {}
|
||||
try {
|
||||
const po = String(process.env.YT_PO_TOKEN || '').trim();
|
||||
if (po) args.push('--extractor-args', `youtube:po_token=${po}`);
|
||||
} catch {}
|
||||
try {
|
||||
const proxy = String(process.env.YT_EGRESS_PROXY || '').trim();
|
||||
if (proxy) args.push('--proxy', proxy);
|
||||
} catch {}
|
||||
return args;
|
||||
}
|
||||
|
||||
export function hasCookiesFile() {
|
||||
try {
|
||||
const c = String(process.env.YT_COOKIES_FILE || '').trim();
|
||||
return Boolean(c && fs.existsSync(c));
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
export function hashSearchKey(parts) {
|
||||
return crypto.createHash('sha256').update(String(parts)).digest('hex').slice(0, 32);
|
||||
}
|
||||
|
||||
// Métriques process-wide (remises à zéro au restart, persistées jour par jour en SQLite via db.mjs)
|
||||
export const ytMetrics = {
|
||||
scrapeCalls: 0,
|
||||
scrapeHits: 0, // hit cache (mémoire ou SQLite)
|
||||
apiCalls: 0,
|
||||
quotaUnits: 0, // estimation search*100 + videos*1
|
||||
fallbacks: 0,
|
||||
scrapeErrors: 0,
|
||||
innertubeCalls: 0,
|
||||
innertubeErrors: 0,
|
||||
};
|
||||
|
||||
export function metricsSnapshot() {
|
||||
return { ...ytMetrics };
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
// Couche InnerTube directe (façon SmartTube/MediaServiceCore) via youtubei.js.
|
||||
// WEB client : search + continuations (pagination illimitée), watch-next (related),
|
||||
// sans clé API ni quota. Le dispatcher bascule sur scrape/API si indisponible.
|
||||
import { hashSearchKey, ytMetrics } from './youtube-common.mjs';
|
||||
|
||||
let sessionPromise = null;
|
||||
let LogSilenced = false;
|
||||
|
||||
async function silenceLibNoise() {
|
||||
if (LogSilenced) return;
|
||||
LogSilenced = true;
|
||||
try {
|
||||
const { Log } = await import('youtubei.js');
|
||||
// youtubei.js loggue en WARN chaque Text sans run assorti (bruit sur les titres) :
|
||||
// on ne garde que les erreurs.
|
||||
if (Log?.set_level && Log?.Level) {
|
||||
const lvl = Log.Level.ERROR ?? Log.Level.WARNING ?? 3;
|
||||
Log.set_level(lvl);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
/** Session InnerTube singleton (lazy). Throw code yt_innertube_unavailable si KO. */
|
||||
export async function getSession() {
|
||||
if (!sessionPromise) {
|
||||
sessionPromise = (async () => {
|
||||
try {
|
||||
await silenceLibNoise();
|
||||
const { Innertube } = await import('youtubei.js');
|
||||
const gl = String(process.env.YT_INNERTUBE_GL || 'FR').trim() || 'FR';
|
||||
const hl = String(process.env.YT_INNERTUBE_HL || 'fr').trim() || 'fr';
|
||||
return await Innertube.create({ lang: hl, location: gl });
|
||||
} catch (e) {
|
||||
sessionPromise = null; // retry au prochain appel
|
||||
throw Object.assign(
|
||||
new Error(`InnerTube indisponible : ${String(e?.message || e).slice(0, 160)}`),
|
||||
{ ytStatus: 502, code: 'yt_innertube_unavailable' },
|
||||
);
|
||||
}
|
||||
})();
|
||||
}
|
||||
return sessionPromise;
|
||||
}
|
||||
|
||||
export function resetSession() { sessionPromise = null; }
|
||||
|
||||
/** "81 973 vues" / "57K" / "1,2 M vues" -> number | undefined. */
|
||||
export function parseViewsText(t) {
|
||||
try {
|
||||
const s = String(t || '').trim();
|
||||
if (!s) return undefined;
|
||||
const m = s.match(/([\d\s\u00a0.,]+)\s*([KMBkmb]|Mds|M|k|B)?/);
|
||||
if (!m) return undefined;
|
||||
const num = Number(m[1].replace(/[\s\u00a0]/g, '').replace(',', '.'));
|
||||
if (!Number.isFinite(num)) return undefined;
|
||||
const suffix = (m[2] || '').toLowerCase();
|
||||
const mult = suffix === 'b' ? 1e9 : suffix === 'm' || suffix === 'mds' ? 1e6 : suffix === 'k' ? 1e3 : 1;
|
||||
const v = Math.round(num * mult);
|
||||
return v > 0 ? v : undefined;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
/** "2 minutes, 27 seconds" / "1 heure, 5 minutes" (label a11y) -> secondes. */
|
||||
export function parseDurationLabel(label) {
|
||||
try {
|
||||
const s = String(label || '').toLowerCase();
|
||||
if (!s) return undefined;
|
||||
const get = (re) => { const m = s.match(re); return m ? Number(m[1]) : 0; };
|
||||
const h = get(/(\d+)\s*(?:hours?|heures?)/);
|
||||
const mnt = get(/(\d+)\s*(?:minutes?)/);
|
||||
const sec = get(/(\d+)\s*(?:seconds?|secondes?)/);
|
||||
const total = h * 3600 + mnt * 60 + sec;
|
||||
return total > 0 ? total : undefined;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
function bestThumb(thumbs) {
|
||||
try {
|
||||
const arr = Array.isArray(thumbs) ? thumbs.filter((t) => t?.url) : [];
|
||||
if (!arr.length) return undefined;
|
||||
return arr[arr.length - 1].url;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Mappe un node InnerTube (Video, CompactVideo, GridVideo, LockupView, ReelItem,
|
||||
* ShortsLockupView, PlaylistPanelVideo, WatchCardCompactVideo) vers Suggestion.
|
||||
* Pur et testable offline. Les LockupView (nouveau renderer YouTube, utilisé
|
||||
* notamment dans le watch-next façon SmartTube) ont une forme imbriquée propre.
|
||||
*/
|
||||
export function mapVideoNode(n) {
|
||||
if (!n || typeof n !== 'object') return null;
|
||||
try {
|
||||
const type = String(n.type || '');
|
||||
if (type === 'LockupView') return mapLockupView(n);
|
||||
if (/playlist|channel|gridchannel|shelf|radio|show|album/i.test(type)
|
||||
&& !/playlistpanelvideo|watchcard/i.test(type)) return null;
|
||||
const id = n.id ? String(n.id) : null;
|
||||
if (!id) return null;
|
||||
const title = n.title?.text ?? (typeof n.title === 'string' ? n.title : '');
|
||||
if (!title) return null;
|
||||
const duration = n.duration?.seconds != null ? Number(n.duration.seconds) : undefined;
|
||||
const author = n.author || n.uploader || null;
|
||||
const authorName = author?.name ?? (typeof author === 'string' ? author : undefined);
|
||||
const authorId = author?.id ? String(author.id) : undefined;
|
||||
const views = parseViewsText(n.view_count?.text ?? n.view_count ?? n.views?.text);
|
||||
const isShort = /short|reel/i.test(type) || (Number.isFinite(duration) && duration > 0 && duration <= 70 && /short/i.test(title) === false && /reel|short/i.test(type));
|
||||
const badges = Array.isArray(n.badges) ? n.badges.map((b) => b?.label).filter(Boolean) : [];
|
||||
return {
|
||||
title: String(title),
|
||||
id,
|
||||
url: `https://www.youtube.com/watch?v=${id}`,
|
||||
thumbnail: bestThumb(n.thumbnails),
|
||||
uploaderName: authorName,
|
||||
type: 'video',
|
||||
...(Number.isFinite(duration) && duration > 0 ? { duration } : {}),
|
||||
...(views !== undefined ? { views } : {}),
|
||||
...(n.published?.text ? { publishedAt: String(n.published.text) } : {}),
|
||||
...(authorId ? { channelId: authorId, channelExternalId: authorId, channelUrl: `https://www.youtube.com/channel/${authorId}` } : {}),
|
||||
...(authorName ? { channelHandle: String(authorName) } : {}),
|
||||
...(n.is_live ? { isLive: true } : {}),
|
||||
...(isShort ? { isShort: true } : {}),
|
||||
...(badges.length ? { badges } : {}),
|
||||
};
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
/** Mappe un LockupView (renderer moderne : watch-next, search, shelves). */
|
||||
export function mapLockupView(n) {
|
||||
try {
|
||||
const ct = String(n.content_type || '').toUpperCase();
|
||||
if (ct && !['VIDEO', 'SHORT', 'MOVIE', 'LIVE'].includes(ct)) return null;
|
||||
const id = n.content_id ? String(n.content_id)
|
||||
: (n.renderer_context?.command_context?.on_tap?.payload?.videoId || null);
|
||||
if (!id) return null;
|
||||
const meta = n.metadata || {};
|
||||
const title = meta.title?.text ?? (typeof meta.title === 'string' ? meta.title : '');
|
||||
if (!title) return null;
|
||||
// Vignette : content_image.image[] (prend la plus large)
|
||||
let thumbnail;
|
||||
try {
|
||||
const imgs = (n.content_image?.image || []).filter((i) => i?.url);
|
||||
thumbnail = imgs.sort((a, b) => (b.width || 0) - (a.width || 0))[0]?.url;
|
||||
} catch {}
|
||||
// Chaîne : 1ère ligne des metadata_rows, sinon label a11y "Go to channel X"
|
||||
let uploaderName;
|
||||
try {
|
||||
const rows = meta.metadata?.metadata_rows || [];
|
||||
const first = rows[0]?.metadata_parts?.[0]?.text;
|
||||
uploaderName = first?.text ?? (typeof first === 'string' ? first : undefined);
|
||||
} catch {}
|
||||
if (!uploaderName) {
|
||||
const a11y = String(meta.image?.a11y_label || '');
|
||||
const m = a11y.match(/^(?:go to channel|aller sur la cha[îi]ne)\s+(.+)$/i);
|
||||
if (m) uploaderName = m[1].trim();
|
||||
}
|
||||
// Vues : 2e ligne ("57K", "1,2 M vues"...), durée : label a11y
|
||||
let views;
|
||||
try {
|
||||
const rows = meta.metadata?.metadata_rows || [];
|
||||
const second = rows[1]?.metadata_parts?.[0]?.text;
|
||||
views = parseViewsText(second?.text ?? second);
|
||||
} catch {}
|
||||
const duration = parseDurationLabel(n.renderer_context?.accessibility_context?.label);
|
||||
return {
|
||||
title: String(title),
|
||||
id,
|
||||
url: `https://www.youtube.com/watch?v=${id}`,
|
||||
thumbnail,
|
||||
uploaderName,
|
||||
type: 'video',
|
||||
...(duration !== undefined ? { duration } : {}),
|
||||
...(views !== undefined ? { views } : {}),
|
||||
...(uploaderName ? { channelHandle: String(uploaderName) } : {}),
|
||||
...(ct === 'SHORT' ? { isShort: true } : {}),
|
||||
...(ct === 'LIVE' ? { isLive: true } : {}),
|
||||
};
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
export function mapNodes(nodes) {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
for (const n of nodes || []) {
|
||||
const m = mapVideoNode(n);
|
||||
if (!m || seen.has(m.id)) continue;
|
||||
seen.add(m.id);
|
||||
out.push(m);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Chaîne de continuations en mémoire : q_hash -> { feed, pages }.
|
||||
// (SQLite persiste les pages déjà servies ; la chaîne évite de rejouer les pages 1..N-1.)
|
||||
const chainCache = new Map();
|
||||
const CHAIN_MAX = 50;
|
||||
function chainGet(k) { const h = chainCache.get(k); if (h) { chainCache.delete(k); chainCache.set(k, h); } return h || null; }
|
||||
function chainSet(k, v) {
|
||||
if (chainCache.has(k)) chainCache.delete(k);
|
||||
chainCache.set(k, v);
|
||||
while (chainCache.size > CHAIN_MAX) { const o = chainCache.keys().next().value; if (o === undefined) break; chainCache.delete(o); }
|
||||
}
|
||||
export function innertubeChainStats() { return { chains: chainCache.size, max: CHAIN_MAX }; }
|
||||
|
||||
function sortParam(sort) {
|
||||
const s = String(sort || 'relevance').toLowerCase();
|
||||
if (s === 'date') return 'upload_date';
|
||||
if (s === 'views') return 'view_count';
|
||||
return 'relevance';
|
||||
}
|
||||
|
||||
/**
|
||||
* Recherche InnerTube avec vraie pagination (continuations).
|
||||
* Page 1 ~35-40 vidéos ; pages suivantes via getContinuation() fusionnées (façon SmartTube).
|
||||
*/
|
||||
export async function searchViaInnerTube(q, opts = {}) {
|
||||
const query = String(q || '').trim();
|
||||
if (query.length < 2) return [];
|
||||
const limit = Math.min(50, Math.max(1, Number(opts?.limit || 24)));
|
||||
const page = Math.min(10, Math.max(1, Number(opts?.page || 1)));
|
||||
const sort = String(opts?.sort || 'relevance');
|
||||
ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
|
||||
const yt = await getSession();
|
||||
const key = `it|${hashSearchKey(`${query.toLowerCase()}|${sort}`)}`;
|
||||
let entry = chainGet(key);
|
||||
let feed = entry?.feed || null;
|
||||
let collected = entry?.items ? [...entry.items] : [];
|
||||
let pagesDone = entry?.pages || 0;
|
||||
// Note : une "page" InnerTube fait ~17-20 vidéos quel que soit `limit`.
|
||||
// On charge donc des continuations jusqu'à couvrir la fenêtre demandée
|
||||
// [start, start+limit[ (façon SmartTube qui remplit son écran au fil des
|
||||
// continuations), au lieu d'aligner 1 page API = 1 page UI.
|
||||
const need = page * limit;
|
||||
try {
|
||||
if (!feed) {
|
||||
feed = await yt.search(query, { sort_by: sortParam(sort) });
|
||||
collected = mapNodes(feed.videos);
|
||||
pagesDone = 1;
|
||||
}
|
||||
let guard = 0;
|
||||
while (collected.length < need && feed?.has_continuation && guard < 12) {
|
||||
guard++;
|
||||
feed = await feed.getContinuation();
|
||||
const fresh = mapNodes(feed.videos).filter((m) => !collected.some((c) => c.id === m.id));
|
||||
collected = collected.concat(fresh);
|
||||
pagesDone++;
|
||||
if (!fresh.length) break;
|
||||
}
|
||||
} catch (e) {
|
||||
if (e?.code === 'yt_innertube_unavailable') throw e;
|
||||
throw Object.assign(new Error(`InnerTube search failed: ${String(e?.message || e).slice(0, 160)}`), { ytStatus: 502, code: 'yt_innertube_failed' });
|
||||
}
|
||||
if (feed) chainSet(key, { feed, items: collected, pages: pagesDone });
|
||||
const start = (page - 1) * limit;
|
||||
return collected.slice(start, start + limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Vidéos connexes façon SmartTube (endpoint watch-next) : ce qui alimente
|
||||
* la colonne "À suivre" sous le lecteur.
|
||||
*/
|
||||
export async function getRelatedViaInnerTube(videoId, limit = 24) {
|
||||
const id = String(videoId || '').trim();
|
||||
if (!id) return [];
|
||||
const n = Math.min(50, Math.max(1, Number(limit || 24)));
|
||||
ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
|
||||
const yt = await getSession();
|
||||
try {
|
||||
const info = await yt.getInfo(id);
|
||||
const feed = info?.watch_next_feed;
|
||||
const arr = Array.isArray(feed) ? feed : (feed ? Array.from(feed) : []);
|
||||
return mapNodes(arr).filter((m) => m.id !== id).slice(0, n);
|
||||
} catch (e) {
|
||||
if (e?.code === 'yt_innertube_unavailable') throw e;
|
||||
throw Object.assign(new Error(`InnerTube related failed: ${String(e?.message || e).slice(0, 160)}`), { ytStatus: 502, code: 'yt_innertube_failed' });
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------- Transcripts via InnerTube --------------------
|
||||
// `getInfo().captions.caption_tracks[]` expose les mêmes URLs timedtext que
|
||||
// yt-dlp découvre via un dump complet (`subtitles`/`automatic_captions`).
|
||||
// On adapte leur forme vers le format yt-dlp pour réutiliser pickTrack(),
|
||||
// orderedTracks() et parseTrackText() de transcript.mjs sans les toucher.
|
||||
|
||||
function normCaptionLang(code) {
|
||||
return String(code || '').trim().toLowerCase().replace(/_/g, '-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapte des caption tracks InnerTube brutes vers un pseudo dump yt-dlp
|
||||
* `{ subtitles, automatic_captions }` (kind 'asr' = auto-généré).
|
||||
* Pur et testable offline.
|
||||
*/
|
||||
export function mapCaptionTracks(captionTracks) {
|
||||
const subtitles = {};
|
||||
const automatic_captions = {};
|
||||
for (const t of captionTracks || []) {
|
||||
const base = String(t?.base_url || '');
|
||||
if (!base) continue;
|
||||
const lang = normCaptionLang(t?.language_code) || 'und';
|
||||
const name = t?.name?.text ?? (typeof t?.name === 'string' ? t.name : lang);
|
||||
const sep = base.includes('?') ? '&' : '?';
|
||||
const track = { url: `${base}${sep}fmt=json3`, ext: 'json3', name: String(name) };
|
||||
const dict = t?.kind === 'asr' ? automatic_captions : subtitles;
|
||||
if (!Array.isArray(dict[lang])) dict[lang] = [];
|
||||
if (!dict[lang].some((x) => x.url === track.url)) dict[lang].push(track);
|
||||
}
|
||||
const languages = Array.from(new Set([...Object.keys(subtitles), ...Object.keys(automatic_captions)]));
|
||||
const trackCount = Object.values(subtitles).concat(Object.values(automatic_captions))
|
||||
.reduce((n, arr) => n + arr.length, 0);
|
||||
return { subtitles, automatic_captions, languages, trackCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* Découvre les pistes de sous-titres d'une vidéo via InnerTube (0 quota,
|
||||
* pas de spawn yt-dlp). Retourne un pseudo dump yt-dlp prêt pour pickTrack().
|
||||
*/
|
||||
export async function getCaptionTracksViaInnerTube(videoId) {
|
||||
const id = String(videoId || '').trim();
|
||||
if (!id) return { subtitles: {}, automatic_captions: {}, languages: [], trackCount: 0 };
|
||||
ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
|
||||
const yt = await getSession();
|
||||
try {
|
||||
const info = await yt.getInfo(id);
|
||||
const raw = info?.captions?.caption_tracks || [];
|
||||
return mapCaptionTracks(Array.isArray(raw) ? raw : Array.from(raw));
|
||||
} catch (e) {
|
||||
if (e?.code === 'yt_innertube_unavailable') throw e;
|
||||
throw Object.assign(new Error(`InnerTube captions failed: ${String(e?.message || e).slice(0, 160)}`), { ytStatus: 502, code: 'yt_innertube_failed' });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
// Recherche / channel / trending YouTube SANS clé API via yt-dlp (Step 17 P1).
|
||||
// Binaire résolu via YT_DLP_PATH > PATH. Jamais de secret en log.
|
||||
import { execFile as execFileCb } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { getYtDlpTimeoutMs, buildYtDlpExtraArgs, resolveYtDlpBin } from './youtube-common.mjs';
|
||||
|
||||
const execFileAsync = promisify(execFileCb);
|
||||
|
||||
/**
|
||||
* Normalise une entrée yt-dlp flat-playlist vers Suggestion (même contrat que youtube.mjs).
|
||||
* @param {any} e entrée yt-dlp
|
||||
*/
|
||||
export function mapFlatEntry(e) {
|
||||
if (!e || typeof e !== 'object') return null;
|
||||
const id = e.id || e.display_id || null;
|
||||
if (!id) return null;
|
||||
const thumbs = Array.isArray(e.thumbnails) ? e.thumbnails : [];
|
||||
const thumb = thumbs.length
|
||||
? (thumbs.find((t) => t?.url && (t.height || 0) >= 360)?.url || thumbs[thumbs.length - 1]?.url)
|
||||
: (e.thumbnail || undefined);
|
||||
const duration = e.duration != null ? Number(e.duration) : undefined;
|
||||
const views = e.view_count != null ? Number(e.view_count) : (e.views != null ? Number(e.views) : undefined);
|
||||
const channelId = e.channel_id || e.uploader_id || undefined;
|
||||
const uploaderName = e.channel || e.uploader || undefined;
|
||||
return {
|
||||
title: e.title || '',
|
||||
id: String(id),
|
||||
url: String(id).startsWith('http') ? String(id) : `https://www.youtube.com/watch?v=${id}`,
|
||||
thumbnail: thumb,
|
||||
uploaderName,
|
||||
type: 'video',
|
||||
...(Number.isFinite(duration) && duration > 0 ? { duration } : {}),
|
||||
...(Number.isFinite(views) ? { views } : {}),
|
||||
...(e.timestamp ? { publishedAt: new Date(Number(e.timestamp) * 1000).toISOString() } : {}),
|
||||
...(channelId ? { channelId: String(channelId), channelExternalId: String(channelId), channelUrl: `https://www.youtube.com/channel/${channelId}` } : {}),
|
||||
...(uploaderName ? { channelHandle: String(uploaderName) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Parse le JSON de `yt-dlp --dump-single-json --flat-playlist` (objet ou NDJSON fog). */
|
||||
export function parseFlatPlaylistJson(raw) {
|
||||
try {
|
||||
const text = String(raw || '').trim();
|
||||
if (!text) return [];
|
||||
// Cas standard : un seul objet JSON avec .entries
|
||||
try {
|
||||
const obj = JSON.parse(text);
|
||||
const entries = Array.isArray(obj?.entries) ? obj.entries : (Array.isArray(obj) ? obj : []);
|
||||
return entries.map(mapFlatEntry).filter(Boolean);
|
||||
} catch {
|
||||
// Fallback NDJSON (une ligne = un JSON)
|
||||
const out = [];
|
||||
for (const line of text.split('\n')) {
|
||||
const t = line.trim();
|
||||
if (!t) continue;
|
||||
try {
|
||||
const o = JSON.parse(t);
|
||||
const m = mapFlatEntry(o);
|
||||
if (m) out.push(m);
|
||||
} catch {}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
export function classifyScrapeError(e) {
|
||||
if (e?.code === 'yt_scrape_no_binary') return e;
|
||||
const msg = String(e?.message || e || '');
|
||||
if (/ENOENT/i.test(msg) || e?.code === 'ENOENT' || e?.errno === 'ENOENT') {
|
||||
return Object.assign(
|
||||
new Error('yt-dlp introuvable (ni PATH ni bundled). Installez yt-dlp ou définissez YT_DLP_PATH.'),
|
||||
{ ytStatus: 503, code: 'yt_scrape_no_binary' },
|
||||
);
|
||||
}
|
||||
if (/bot|sign in to confirm|challenge|cookies|login/i.test(msg)) {
|
||||
return Object.assign(new Error('YouTube bot-check (cookies/PO-Token requis)'), { ytStatus: 502, code: 'yt_scrape_bot_check' });
|
||||
}
|
||||
if (/timed out|timeout|ETIMEDOUT|killed|SIGKILL/i.test(msg)) {
|
||||
return Object.assign(new Error('YouTube scrape timeout'), { ytStatus: 504, code: 'yt_scrape_timeout' });
|
||||
}
|
||||
if (/unable to|unsupported|not available|private|deleted/i.test(msg)) {
|
||||
return Object.assign(new Error(`YouTube scrape upstream: ${msg.slice(0, 160)}`), { ytStatus: 502, code: 'yt_scrape_upstream' });
|
||||
}
|
||||
return Object.assign(new Error(`YouTube scrape failed: ${msg.slice(0, 200)}`), { ytStatus: 502, code: 'yt_scrape_failed' });
|
||||
}
|
||||
|
||||
async function runYtDlpFlat(queryOrUrl, { limit = 10, playlistStart = 1 } = {}) {
|
||||
const bin = await resolveYtDlpBin();
|
||||
const timeout = getYtDlpTimeoutMs();
|
||||
const end = playlistStart + Math.max(1, Math.min(50, Number(limit || 10))) - 1;
|
||||
const args = [
|
||||
'--dump-single-json',
|
||||
'--flat-playlist',
|
||||
'--no-warnings',
|
||||
'--no-check-certificates',
|
||||
'--skip-download',
|
||||
'--no-playlist',
|
||||
'--playlist-start', String(playlistStart),
|
||||
'--playlist-end', String(end),
|
||||
...buildYtDlpExtraArgs(),
|
||||
queryOrUrl,
|
||||
];
|
||||
try {
|
||||
const { stdout } = await execFileAsync(bin, args, { timeout, maxBuffer: 16 * 1024 * 1024 });
|
||||
return parseFlatPlaylistJson(stdout);
|
||||
} catch (e) {
|
||||
// yt-dlp peut écrire du JSON partiel sur stdout même en exit non-zero
|
||||
const partial = e?.stdout ? parseFlatPlaylistJson(String(e.stdout)) : [];
|
||||
if (partial.length) return partial;
|
||||
throw classifyScrapeError(e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Recherche sans clé. sort: relevance|date|views */
|
||||
export async function searchViaScrape(q, opts = {}) {
|
||||
const query = String(q || '').trim();
|
||||
if (query.length < 2) return [];
|
||||
const limit = Math.min(50, Math.max(1, Number(opts?.limit || 10)));
|
||||
const page = Math.max(1, Number(opts?.page || 1));
|
||||
const sort = String(opts?.sort || 'relevance').toLowerCase();
|
||||
const perPage = limit;
|
||||
const start = (page - 1) * perPage + 1;
|
||||
let prefix = 'ytsearch';
|
||||
if (sort === 'date') prefix = 'ytsearchdate';
|
||||
// views : pas de préfixe natif stable -> ytsearch + tri local
|
||||
const target = `${prefix}${perPage}:${query}`;
|
||||
let items = await runYtDlpFlat(target, { limit: perPage, playlistStart: start });
|
||||
if (sort === 'views') items = [...items].sort((a, b) => (b.views || 0) - (a.views || 0));
|
||||
return items.slice(0, perPage);
|
||||
}
|
||||
|
||||
const CHANNEL_TABS = { videos: 'videos', shorts: 'shorts', streams: 'streams', live: 'streams', playlists: 'playlists' };
|
||||
|
||||
/** Contenu chaîne sans clé. type: videos|shorts|playlists|live */
|
||||
export async function fetchChannelViaScrape(externalId, { type = 'videos', page = 1, limit = 24 } = {}) {
|
||||
const raw = String(externalId || '').trim().replace(/^@/, '');
|
||||
if (!raw) return { items: [], nextPage: null };
|
||||
const tab = CHANNEL_TABS[String(type)] || 'videos';
|
||||
const perPage = Math.min(50, Math.max(1, Number(limit || 24)));
|
||||
const pageNum = Math.max(1, Number(page || 1));
|
||||
const start = (pageNum - 1) * perPage + 1;
|
||||
// UC... -> /channel/UC.../tab, sinon -> /@/handle/tab
|
||||
const base = /^UC[\w-]{20,}$/.test(raw)
|
||||
? `https://www.youtube.com/channel/${raw}/${tab}`
|
||||
: `https://www.youtube.com/@${raw}/${tab}`;
|
||||
const items = await runYtDlpFlat(base, { limit: perPage, playlistStart: start });
|
||||
// playlists renvoient des ids PL... : on les garde tels quels avec un shape playlist
|
||||
if (tab === 'playlists') {
|
||||
const mapped = items.map((s) => ({ id: s.id, title: s.title, thumbnail: s.thumbnail, videoCount: null, updatedAt: s.publishedAt || null }));
|
||||
return { items: mapped, nextPage: items.length >= perPage ? pageNum + 1 : null };
|
||||
}
|
||||
let filtered = items;
|
||||
if (type === 'shorts') filtered = items.filter((s) => !s.duration || s.duration <= 70);
|
||||
return { items: filtered, nextPage: items.length >= perPage ? pageNum + 1 : null };
|
||||
}
|
||||
|
||||
/** Trending sans clé. Les onglets /feed/trending ont été retirés côté YouTube
|
||||
* (redirect home -> erreur tab) ; on tente les tabs puis repli ytsearch trié par vues. */
|
||||
export async function getTrendingViaScrape(limit = 24) {
|
||||
const n = Math.min(50, Math.max(1, Number(limit || 24)));
|
||||
const tabs = [
|
||||
'https://www.youtube.com/feed/trending',
|
||||
'https://www.youtube.com/trending',
|
||||
];
|
||||
for (const url of tabs) {
|
||||
try {
|
||||
const items = await runYtDlpFlat(url, { limit: n, playlistStart: 1 });
|
||||
if (items.length) return items;
|
||||
} catch {}
|
||||
}
|
||||
// Repli : recherche générique triée par vues (0 quota, toujours disponible)
|
||||
return runYtDlpFlat(`ytsearch${n}:top trending videos world`, { limit: n, playlistStart: 1 });
|
||||
}
|
||||
|
||||
/** Résout un channel_id via scrape (imprime channel_id sans appel API). */
|
||||
export async function resolveChannelIdViaScrape(externalId) {
|
||||
const raw = String(externalId || '').trim();
|
||||
if (/^UC[\w-]{20,}$/.test(raw)) return raw;
|
||||
const handle = raw.replace(/^@/, '');
|
||||
const bin = await resolveYtDlpBin().catch(() => null);
|
||||
if (!bin) {
|
||||
throw Object.assign(
|
||||
new Error('yt-dlp introuvable (ni PATH ni bundled). Installez yt-dlp ou définissez YT_DLP_PATH.'),
|
||||
{ ytStatus: 503, code: 'yt_scrape_no_binary' },
|
||||
);
|
||||
}
|
||||
const timeout = Math.min(15000, getYtDlpTimeoutMs());
|
||||
try {
|
||||
const { stdout } = await execFileAsync(bin,
|
||||
['--print', '%(channel_id)s', '--no-warnings', '--skip-download', '--playlist-end', '1', ...buildYtDlpExtraArgs(), `https://www.youtube.com/@${handle}/videos`],
|
||||
{ timeout });
|
||||
const id = String(stdout || '').trim().split('\n')[0].trim();
|
||||
if (/^UC[\w-]{20,}$/.test(id)) return id;
|
||||
} catch {}
|
||||
return raw;
|
||||
}
|
||||
+268
-143
@@ -9,6 +9,12 @@
|
||||
* @property {string=} uploaderName
|
||||
* @property {string=} type
|
||||
*/
|
||||
import {
|
||||
getYouTubeKeys, isKeyFailure, getSearchMode, getScrapeTtlMs,
|
||||
hashSearchKey, ytMetrics,
|
||||
} from './youtube-common.mjs';
|
||||
import { searchViaScrape } from './youtube-scrape.mjs';
|
||||
import { searchViaInnerTube } from './youtube-innertube.mjs';
|
||||
|
||||
function parseISODurationToSeconds(iso) {
|
||||
if (typeof iso !== 'string' || !iso) return 0;
|
||||
@@ -20,50 +26,6 @@ function parseISODurationToSeconds(iso) {
|
||||
return (hours * 3600) + (minutes * 60) + seconds;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve configured YouTube API keys.
|
||||
* Accepts YOUTUBE_API_KEYS as JSON array ('["k1","k2"]') or CSV ('k1,k2'),
|
||||
* plus the legacy single YOUTUBE_API_KEY as fallback. De-duplicated.
|
||||
* @returns {string[]}
|
||||
*/
|
||||
function getYouTubeKeys() {
|
||||
const keys = [];
|
||||
try {
|
||||
const raw = process.env.YOUTUBE_API_KEYS;
|
||||
if (raw && String(raw).trim() && String(raw).trim() !== 'undefined' && String(raw).trim() !== 'null') {
|
||||
const s = String(raw).trim();
|
||||
if (s.startsWith('[')) {
|
||||
try {
|
||||
const arr = JSON.parse(s);
|
||||
if (Array.isArray(arr)) keys.push(...arr.map(v => String(v || '').trim()).filter(Boolean));
|
||||
} catch {}
|
||||
} else {
|
||||
keys.push(...s.split(',').map(v => String(v || '').trim()).filter(Boolean));
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
try {
|
||||
const single = process.env.YOUTUBE_API_KEY;
|
||||
if (single && String(single).trim()) keys.push(String(single).trim());
|
||||
} catch {}
|
||||
return Array.from(new Set(keys.filter(Boolean)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a YouTube API failure as retryable with another key.
|
||||
* - 400 API_KEY_INVALID / "API key expired" -> key is dead, try next.
|
||||
* - 403 quotaExceeded / rateLimitExceeded / dailyLimitExceeded -> quota, try next.
|
||||
*/
|
||||
function isKeyFailure(status, data) {
|
||||
try {
|
||||
const reason = data?.error?.errors?.[0]?.reason || '';
|
||||
const message = String(data?.error?.message || '');
|
||||
if (status === 400 && (reason === 'API_KEY_INVALID' || /api key (expired|invalid)/i.test(message))) return true;
|
||||
if (status === 403 && /quota|rateLimit|dailyLimit|userRateLimit/i.test(reason + ' ' + message)) return true;
|
||||
} catch {}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** GET JSON from YouTube, rotating through keys on key failures. */
|
||||
async function ytFetchJson(base, paramsWithoutKey) {
|
||||
const keys = getYouTubeKeys();
|
||||
@@ -76,13 +38,19 @@ async function ytFetchJson(base, paramsWithoutKey) {
|
||||
for (const key of keys) {
|
||||
const params = new URLSearchParams(paramsWithoutKey);
|
||||
params.set('key', key);
|
||||
ytMetrics.apiCalls++;
|
||||
try {
|
||||
const { incYoutubeMetrics } = await import('../db.mjs').catch(() => ({}));
|
||||
if (String(base).includes('/search')) incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 100 });
|
||||
else incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 1 });
|
||||
ytMetrics.quotaUnits += String(base).includes('/search') ? 100 : 1;
|
||||
} catch {}
|
||||
const resp = await fetch(`${base}?${params.toString()}`);
|
||||
const data = await resp.json().catch(() => ({}));
|
||||
if (resp.ok) return data;
|
||||
lastStatus = resp.status;
|
||||
lastData = data;
|
||||
lastError = new Error(`YouTube API error: ${resp.status} ${data?.error?.message || ''}`.trim());
|
||||
// Only rotate to the next key on key-attributable failures; otherwise fail fast.
|
||||
if (!isKeyFailure(resp.status, data)) break;
|
||||
console.warn(`[YouTube] key ...${String(key).slice(-4)} failed (${resp.status}), trying next key`);
|
||||
}
|
||||
@@ -92,121 +60,278 @@ async function ytFetchJson(base, paramsWithoutKey) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a `suggestqueries.google.com` (client=youtube) response body into plain strings.
|
||||
* The endpoint answers either bare JSON (`["q",["s1","s2"],...]`) or wrapped in
|
||||
* `window.google.ac.h(...)`. Never throws: unparsable bodies yield [].
|
||||
* @param {string} text
|
||||
* @returns {string[]}
|
||||
*/
|
||||
export function parseYoutubeSuggestResponse(text) {
|
||||
try {
|
||||
const raw = String(text || '').trim();
|
||||
if (!raw) return [];
|
||||
let payload = raw;
|
||||
const start = raw.indexOf('(');
|
||||
const end = raw.lastIndexOf(')');
|
||||
if (start !== -1 && end > start) payload = raw.slice(start + 1, end);
|
||||
const parsed = JSON.parse(payload);
|
||||
const candidates = Array.isArray(parsed) && Array.isArray(parsed[1]) ? parsed[1] : [];
|
||||
return candidates
|
||||
.map((s) => {
|
||||
if (Array.isArray(s)) s = s[0];
|
||||
return (typeof s === 'string' ? s : String(s ?? '')).trim();
|
||||
})
|
||||
.filter(Boolean)
|
||||
.slice(0, 20);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Mémoire LRU process-wide pour le dispatcher (complète SQLite)
|
||||
const memCache = new Map();
|
||||
const MEM_MAX = Number(process.env.YT_SCRAPE_MEM_MAX || 300);
|
||||
function memGet(k, ttl) {
|
||||
const hit = memCache.get(k);
|
||||
if (!hit) return null;
|
||||
if ((Date.now() - hit.ts) >= ttl) { memCache.delete(k); return null; }
|
||||
memCache.delete(k); memCache.set(k, hit);
|
||||
return hit.items;
|
||||
}
|
||||
function memSet(k, items) {
|
||||
if (memCache.has(k)) memCache.delete(k);
|
||||
memCache.set(k, { ts: Date.now(), items });
|
||||
while (memCache.size > MEM_MAX) { const o = memCache.keys().next().value; if (o === undefined) break; memCache.delete(o); }
|
||||
}
|
||||
export function ytScrapeCacheStats() {
|
||||
return { memEntries: memCache.size, memMax: MEM_MAX };
|
||||
}
|
||||
|
||||
async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance' } = {}) {
|
||||
const keys = getYouTubeKeys();
|
||||
if (!keys.length) {
|
||||
throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' });
|
||||
}
|
||||
let order = 'relevance';
|
||||
if (sort === 'date') order = 'date';
|
||||
else if (sort === 'views') order = 'viewCount';
|
||||
const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
|
||||
const targetPage = Math.max(1, Number(page || 1));
|
||||
let pageToken = '';
|
||||
let currentPage = 1;
|
||||
let lastItems = [];
|
||||
while (currentPage <= targetPage) {
|
||||
const params = {
|
||||
part: 'snippet', q, type: 'video', maxResults: String(perPage), order,
|
||||
videoEmbeddable: 'true', safeSearch: 'moderate',
|
||||
};
|
||||
if (pageToken) params.pageToken = pageToken;
|
||||
const data = await ytFetchJson('https://www.googleapis.com/youtube/v3/search', params);
|
||||
if (currentPage === targetPage) { lastItems = Array.isArray(data.items) ? data.items : []; break; }
|
||||
const next = data.nextPageToken;
|
||||
if (!next) { lastItems = []; break; }
|
||||
pageToken = String(next);
|
||||
currentPage++;
|
||||
}
|
||||
const videoIds = (lastItems || []).map((item) => item?.id?.videoId).filter(Boolean);
|
||||
const detailsMap = new Map();
|
||||
if (videoIds.length > 0) {
|
||||
try {
|
||||
const detailsData = await ytFetchJson('https://www.googleapis.com/youtube/v3/videos', {
|
||||
part: 'contentDetails,statistics,status', id: videoIds.join(','),
|
||||
});
|
||||
for (const vid of detailsData?.items || []) if (vid?.id) detailsMap.set(vid.id, vid);
|
||||
} catch (e) {
|
||||
console.warn('[YouTube] details fetch failed, continuing without durations:', e?.message || e);
|
||||
}
|
||||
}
|
||||
return (lastItems || []).map((item) => {
|
||||
const videoId = item?.id?.videoId;
|
||||
const snippet = item?.snippet || {};
|
||||
const thumb = snippet.thumbnails?.high?.url || snippet.thumbnails?.medium?.url || snippet.thumbnails?.default?.url || undefined;
|
||||
const details = videoId ? detailsMap.get(videoId) : null;
|
||||
const duration = parseISODurationToSeconds(details?.contentDetails?.duration || '');
|
||||
const views = details?.statistics?.viewCount != null ? Number(details.statistics.viewCount) : undefined;
|
||||
const channelId = snippet.channelId || undefined;
|
||||
const embeddable = details?.status ? details.status.embeddable !== false : undefined;
|
||||
return {
|
||||
title: snippet.title || '', id: videoId,
|
||||
url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined,
|
||||
thumbnail: thumb, uploaderName: snippet.channelTitle || undefined, type: 'video',
|
||||
duration: duration > 0 ? duration : undefined, views,
|
||||
publishedAt: snippet.publishedAt || undefined, channelId,
|
||||
channelHandle: snippet.channelTitle || undefined, channelExternalId: channelId,
|
||||
channelUrl: channelId ? `https://www.youtube.com/channel/${channelId}` : undefined, embeddable,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** @type {{ id: 'yt', label: string, search: (q: string, opts: { limit: number, page?: number, sort?: 'relevance'|'date'|'views' }) => Promise<Suggestion[]> }} */
|
||||
const handler = {
|
||||
id: 'yt',
|
||||
label: 'YouTube',
|
||||
async search(q, opts) {
|
||||
const { limit = 10, page = 1, sort = 'relevance' } = opts || {};
|
||||
const mode = getSearchMode();
|
||||
const ttl = getScrapeTtlMs();
|
||||
const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
|
||||
const key = `yt|${hashSearchKey(`${String(q).toLowerCase().trim()}|${perPage}|${page}|${sort}|${mode}`)}`;
|
||||
// 1) mémoire
|
||||
const memHit = memGet(key, ttl);
|
||||
if (memHit) { ytMetrics.scrapeHits++; return memHit; }
|
||||
// 2) SQLite
|
||||
try {
|
||||
const keys = getYouTubeKeys();
|
||||
if (!keys.length) {
|
||||
throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' });
|
||||
const { getCachedYoutubeSearch } = await import('../db.mjs');
|
||||
const cached = getCachedYoutubeSearch(key);
|
||||
if (cached?.items) {
|
||||
ytMetrics.scrapeHits++;
|
||||
memSet(key, cached.items);
|
||||
return cached.items;
|
||||
}
|
||||
|
||||
let order = 'relevance';
|
||||
if (sort === 'date') order = 'date';
|
||||
else if (sort === 'views') order = 'viewCount';
|
||||
|
||||
// Iterate nextPageToken to reach the requested page (1-based)
|
||||
const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
|
||||
const targetPage = Math.max(1, Number(page || 1));
|
||||
let pageToken = '';
|
||||
let currentPage = 1;
|
||||
let lastItems = [];
|
||||
|
||||
while (currentPage <= targetPage) {
|
||||
const params = {
|
||||
part: 'snippet',
|
||||
q: q,
|
||||
type: 'video',
|
||||
maxResults: String(perPage),
|
||||
order,
|
||||
// Ne retourner que des vidéos lisibles en embed (évite l'erreur 153 côté player).
|
||||
// Les vidéos non-embeddables sont filtrées via le champ status ci-dessous pour /videos.
|
||||
videoEmbeddable: 'true',
|
||||
safeSearch: 'moderate',
|
||||
};
|
||||
if (pageToken) params.pageToken = pageToken;
|
||||
|
||||
const data = await ytFetchJson('https://www.googleapis.com/youtube/v3/search', params);
|
||||
|
||||
if (currentPage === targetPage) {
|
||||
lastItems = Array.isArray(data.items) ? data.items : [];
|
||||
break;
|
||||
}
|
||||
|
||||
// Prepare for next iteration
|
||||
const next = data.nextPageToken;
|
||||
if (!next) {
|
||||
// No more pages; requested page beyond available results
|
||||
lastItems = [];
|
||||
break;
|
||||
}
|
||||
pageToken = String(next);
|
||||
currentPage++;
|
||||
} catch {}
|
||||
const persist = async (items, source) => {
|
||||
// Ne jamais persister un résultat vide : une page vide transitoire
|
||||
// (continuation expirée, raté réseau partiel) ne doit pas empoisonner
|
||||
// le cache et bloquer les requêtes suivantes.
|
||||
if (!Array.isArray(items) || items.length === 0) return items;
|
||||
memSet(key, items);
|
||||
try {
|
||||
const { setCachedYoutubeSearch, incYoutubeMetrics } = await import('../db.mjs');
|
||||
setCachedYoutubeSearch(key, q, items, source, ttl);
|
||||
if (source === 'scrape') incYoutubeMetrics({ scrapeCalls: 1 });
|
||||
} catch {}
|
||||
return items;
|
||||
};
|
||||
const t0 = Date.now();
|
||||
const tryScrape = async () => {
|
||||
ytMetrics.scrapeCalls++;
|
||||
try { const { incYoutubeMetrics } = await import('../db.mjs'); incYoutubeMetrics({ scrapeCalls: 1 }); } catch {}
|
||||
return searchViaScrape(q, { limit: perPage, page, sort });
|
||||
};
|
||||
const tryApi = () => searchViaApi(q, { limit: perPage, page, sort });
|
||||
const tryInnerTube = () => searchViaInnerTube(q, { limit: perPage, page, sort });
|
||||
const log = (source, extra = '') => console.log(`[YT search] source=${source} mode=${mode} latency=${Date.now() - t0}ms results=${extra}`);
|
||||
try {
|
||||
if (mode === 'api-only') {
|
||||
const items = await tryApi();
|
||||
log('api', items.length);
|
||||
return persist(items, 'api');
|
||||
}
|
||||
|
||||
const videoIds = (lastItems || [])
|
||||
.map(item => item?.id?.videoId)
|
||||
.filter(Boolean);
|
||||
|
||||
const detailsMap = new Map();
|
||||
if (videoIds.length > 0) {
|
||||
if (mode === 'scrape-only') {
|
||||
const items = await tryScrape();
|
||||
log('scrape', items.length);
|
||||
return persist(items, 'scrape');
|
||||
}
|
||||
if (mode === 'innertube-only') {
|
||||
const items = await tryInnerTube();
|
||||
log('innertube', items.length);
|
||||
return persist(items, 'innertube');
|
||||
}
|
||||
if (mode === 'api-first') {
|
||||
try {
|
||||
const detailsData = await ytFetchJson('https://www.googleapis.com/youtube/v3/videos', {
|
||||
part: 'contentDetails,statistics,status',
|
||||
id: videoIds.join(','),
|
||||
});
|
||||
for (const vid of detailsData?.items || []) {
|
||||
if (vid?.id) detailsMap.set(vid.id, vid);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[YouTube] details fetch failed, continuing without durations:', e?.message || e);
|
||||
const items = await tryApi();
|
||||
log('api', items.length);
|
||||
return persist(items, 'api');
|
||||
} catch (apiErr) {
|
||||
ytMetrics.fallbacks++;
|
||||
console.warn(`[YT search] api-first fallback to scrape: ${apiErr?.message || apiErr}`);
|
||||
const items = await tryScrape();
|
||||
log('scrape(fallback)', items.length);
|
||||
return persist(items, 'scrape');
|
||||
}
|
||||
}
|
||||
|
||||
return (lastItems || []).map(item => {
|
||||
const videoId = item?.id?.videoId;
|
||||
const snippet = item?.snippet || {};
|
||||
const thumb = snippet.thumbnails?.high?.url
|
||||
|| snippet.thumbnails?.medium?.url
|
||||
|| snippet.thumbnails?.default?.url
|
||||
|| undefined;
|
||||
|
||||
const details = videoId ? detailsMap.get(videoId) : null;
|
||||
const isoDuration = details?.contentDetails?.duration || '';
|
||||
const duration = parseISODurationToSeconds(isoDuration);
|
||||
const views = details?.statistics?.viewCount != null ? Number(details.statistics.viewCount) : undefined;
|
||||
const channelId = snippet.channelId || undefined;
|
||||
const channelHandle = snippet.channelTitle || undefined;
|
||||
// status.embeddable === false -> le player renvoie l'erreur 153 ("Video configuration error").
|
||||
const embeddable = details?.status ? details.status.embeddable !== false : undefined;
|
||||
|
||||
return {
|
||||
title: snippet.title || '',
|
||||
id: videoId,
|
||||
url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined,
|
||||
thumbnail: thumb,
|
||||
uploaderName: snippet.channelTitle || undefined,
|
||||
type: 'video',
|
||||
duration: duration > 0 ? duration : undefined,
|
||||
views,
|
||||
publishedAt: snippet.publishedAt || undefined,
|
||||
channelId,
|
||||
channelHandle,
|
||||
channelExternalId: channelId,
|
||||
channelUrl: channelId ? `https://www.youtube.com/channel/${channelId}` : undefined,
|
||||
embeddable,
|
||||
};
|
||||
});
|
||||
if (mode === 'scrape-first') {
|
||||
// scrape -> API (comportement historique, sans InnerTube)
|
||||
try {
|
||||
const items = await tryScrape();
|
||||
log('scrape', items.length);
|
||||
return persist(items, 'scrape');
|
||||
} catch (scrapeErr) {
|
||||
ytMetrics.scrapeErrors++;
|
||||
ytMetrics.fallbacks++;
|
||||
console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`);
|
||||
try {
|
||||
const items = await tryApi();
|
||||
log('api(fallback)', items.length);
|
||||
return persist(items, 'api');
|
||||
} catch (apiErr) {
|
||||
const noBinary = scrapeErr?.code === 'yt_scrape_no_binary';
|
||||
const noKey = apiErr?.code === 'youtube_api_key_unavailable';
|
||||
if (noBinary && noKey) {
|
||||
const err = new Error('YouTube indisponible : yt-dlp introuvable (définissez YT_DLP_PATH) et aucune clé YOUTUBE_API_KEY configurée.');
|
||||
err.ytStatus = 503; err.code = 'youtube_no_source';
|
||||
throw err;
|
||||
}
|
||||
console.error(`[YT search] both sources failed. scrape=${scrapeErr?.message} api=${apiErr?.message}`);
|
||||
throw scrapeErr;
|
||||
}
|
||||
}
|
||||
}
|
||||
// innertube-first (défaut) : InnerTube -> scrape yt-dlp -> API officielle.
|
||||
// Chaque couche ne fait jamais échouer la recherche à elle seule.
|
||||
const errors = [];
|
||||
try {
|
||||
const items = await tryInnerTube();
|
||||
log('innertube', items.length);
|
||||
return persist(items, 'innertube');
|
||||
} catch (itErr) {
|
||||
ytMetrics.innertubeErrors = (ytMetrics.innertubeErrors || 0) + 1;
|
||||
ytMetrics.fallbacks++;
|
||||
errors.push(`innertube=${itErr?.message}`);
|
||||
console.warn(`[YT search] innertube failed (${itErr?.code || 'unknown'}), fallback to scrape`);
|
||||
}
|
||||
try {
|
||||
const items = await tryScrape();
|
||||
log('scrape(fallback)', items.length);
|
||||
return persist(items, 'scrape');
|
||||
} catch (scrapeErr) {
|
||||
ytMetrics.scrapeErrors++;
|
||||
ytMetrics.fallbacks++;
|
||||
errors.push(`scrape=${scrapeErr?.message}`);
|
||||
console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`);
|
||||
}
|
||||
try {
|
||||
const items = await tryApi();
|
||||
log('api(fallback)', items.length);
|
||||
return persist(items, 'api');
|
||||
} catch (apiErr) {
|
||||
errors.push(`api=${apiErr?.message}`);
|
||||
const noSource = !getYouTubeKeys().length;
|
||||
if (noSource) {
|
||||
const err = new Error(`YouTube indisponible (${errors.join(' | ').slice(0, 220)}). Vérifiez le réseau/YT_DLP_PATH ou ajoutez YOUTUBE_API_KEYS.`);
|
||||
err.ytStatus = 503; err.code = 'youtube_no_source';
|
||||
throw err;
|
||||
}
|
||||
console.error(`[YT search] all sources failed. ${errors.join(' | ')}`);
|
||||
const err = new Error(errors[0] || 'search_failed');
|
||||
err.ytStatus = 502; err.code = 'youtube_all_sources_failed';
|
||||
throw err;
|
||||
}
|
||||
} catch (error) {
|
||||
// Log explicite (clé expirée / quota) puis remonte l'erreur pour que
|
||||
// /api/search la reporte dans `errors.yt` au lieu d'un groupe vide silencieux.
|
||||
const status = error?.ytStatus || 500;
|
||||
console.error(`YouTube search error (status ${status}):`, error?.message || error);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
async suggest(q, opts) {
|
||||
const limit = Math.min(20, Math.max(1, Number(opts?.limit || 10)));
|
||||
const query = String(q || '').trim();
|
||||
if (query.length < 2) return [];
|
||||
try {
|
||||
const params = new URLSearchParams({ client: 'youtube', ds: 'yt', hl: 'fr', q: query });
|
||||
const headers = { 'User-Agent': 'Mozilla/5.0' };
|
||||
const proxy = String(process.env.YT_EGRESS_PROXY || '').trim();
|
||||
const fetchOpts = { headers, signal: AbortSignal.timeout(6000) };
|
||||
// Note : suggest reste sans proxy (endpoint Google public léger) sauf si egress configuré côté infra.
|
||||
if (proxy) console.debug?.('[YT suggest] egress proxy configured, direct fetch kept for suggest');
|
||||
const resp = await fetch(`https://suggestqueries.google.com/complete/search?${params.toString()}`, fetchOpts);
|
||||
if (!resp.ok) return [];
|
||||
const text = await resp.text();
|
||||
return parseYoutubeSuggestResponse(text).slice(0, limit);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user