feat(youtube): InnerTube-first search, transcripts and watch-next related (Steps 15-18)

- InnerTube layer via pinned youtubei.js 18.1.0 (no quota, no key): search with merged continuations (unlimited pages), watch-next related with LockupView mapping, caption-track discovery - 3-layer dispatcher (YT_SEARCH_MODE, default innertube-first): innertube -> yt-dlp scrape -> official API, graceful errors.yt - Robust yt-dlp binary resolution (YT_DLP_PATH > PATH > bundled) with systematic API fallback (fixes spawn ENOENT in UI) - Transcript: InnerTube caption discovery (YT_TRANSCRIPT_SOURCE), reusing pickTrack/orderedTracks/parseTrackText; yt-dlp fallback kept - Watch: sidebar uses real watch-next related[] (/api/details), title-search fallback for other providers - Cache: memory LRU + SQLite (youtube_search_cache, youtube_metrics), never persist empty pages; /healthz observability; /api/trending - Includes pending Step 15/16 leftovers in same files (suggest, test scripts); unrelated provider adapters left uncommitted
2026-09-25 19:38:46 -04:00
parent 300e5c45b1
commit 4bdcd393fe
25 changed files with 4112 additions and 168 deletions
@@ -0,0 +1,331 @@
+// Couche InnerTube directe (façon SmartTube/MediaServiceCore) via youtubei.js.
+// WEB client : search + continuations (pagination illimitée), watch-next (related),
+// sans clé API ni quota. Le dispatcher bascule sur scrape/API si indisponible.
+import { hashSearchKey, ytMetrics } from './youtube-common.mjs';
+
+let sessionPromise = null;
+let LogSilenced = false;
+
+async function silenceLibNoise() {
+  if (LogSilenced) return;
+  LogSilenced = true;
+  try {
+    const { Log } = await import('youtubei.js');
+    // youtubei.js loggue en WARN chaque Text sans run assorti (bruit sur les titres) :
+    // on ne garde que les erreurs.
+    if (Log?.set_level && Log?.Level) {
+      const lvl = Log.Level.ERROR ?? Log.Level.WARNING ?? 3;
+      Log.set_level(lvl);
+    }
+  } catch {}
+}
+
+/** Session InnerTube singleton (lazy). Throw code yt_innertube_unavailable si KO. */
+export async function getSession() {
+  if (!sessionPromise) {
+    sessionPromise = (async () => {
+      try {
+        await silenceLibNoise();
+        const { Innertube } = await import('youtubei.js');
+        const gl = String(process.env.YT_INNERTUBE_GL || 'FR').trim() || 'FR';
+        const hl = String(process.env.YT_INNERTUBE_HL || 'fr').trim() || 'fr';
+        return await Innertube.create({ lang: hl, location: gl });
+      } catch (e) {
+        sessionPromise = null; // retry au prochain appel
+        throw Object.assign(
+          new Error(`InnerTube indisponible : ${String(e?.message || e).slice(0, 160)}`),
+          { ytStatus: 502, code: 'yt_innertube_unavailable' },
+        );
+      }
+    })();
+  }
+  return sessionPromise;
+}
+
+export function resetSession() { sessionPromise = null; }
+
+/** "81 973 vues" / "57K" / "1,2 M vues" -> number | undefined. */
+export function parseViewsText(t) {
+  try {
+    const s = String(t || '').trim();
+    if (!s) return undefined;
+    const m = s.match(/([\d\s\u00a0.,]+)\s*([KMBkmb]|Mds|M|k|B)?/);
+    if (!m) return undefined;
+    const num = Number(m[1].replace(/[\s\u00a0]/g, '').replace(',', '.'));
+    if (!Number.isFinite(num)) return undefined;
+    const suffix = (m[2] || '').toLowerCase();
+    const mult = suffix === 'b' ? 1e9 : suffix === 'm' || suffix === 'mds' ? 1e6 : suffix === 'k' ? 1e3 : 1;
+    const v = Math.round(num * mult);
+    return v > 0 ? v : undefined;
+  } catch { return undefined; }
+}
+
+/** "2 minutes, 27 seconds" / "1 heure, 5 minutes" (label a11y) -> secondes. */
+export function parseDurationLabel(label) {
+  try {
+    const s = String(label || '').toLowerCase();
+    if (!s) return undefined;
+    const get = (re) => { const m = s.match(re); return m ? Number(m[1]) : 0; };
+    const h = get(/(\d+)\s*(?:hours?|heures?)/);
+    const mnt = get(/(\d+)\s*(?:minutes?)/);
+    const sec = get(/(\d+)\s*(?:seconds?|secondes?)/);
+    const total = h * 3600 + mnt * 60 + sec;
+    return total > 0 ? total : undefined;
+  } catch { return undefined; }
+}
+
+function bestThumb(thumbs) {
+  try {
+    const arr = Array.isArray(thumbs) ? thumbs.filter((t) => t?.url) : [];
+    if (!arr.length) return undefined;
+    return arr[arr.length - 1].url;
+  } catch { return undefined; }
+}
+
+/**
+ * Mappe un node InnerTube (Video, CompactVideo, GridVideo, LockupView, ReelItem,
+ * ShortsLockupView, PlaylistPanelVideo, WatchCardCompactVideo) vers Suggestion.
+ * Pur et testable offline. Les LockupView (nouveau renderer YouTube, utilisé
+ * notamment dans le watch-next façon SmartTube) ont une forme imbriquée propre.
+ */
+export function mapVideoNode(n) {
+  if (!n || typeof n !== 'object') return null;
+  try {
+    const type = String(n.type || '');
+    if (type === 'LockupView') return mapLockupView(n);
+    if (/playlist|channel|gridchannel|shelf|radio|show|album/i.test(type)
+      && !/playlistpanelvideo|watchcard/i.test(type)) return null;
+    const id = n.id ? String(n.id) : null;
+    if (!id) return null;
+    const title = n.title?.text ?? (typeof n.title === 'string' ? n.title : '');
+    if (!title) return null;
+    const duration = n.duration?.seconds != null ? Number(n.duration.seconds) : undefined;
+    const author = n.author || n.uploader || null;
+    const authorName = author?.name ?? (typeof author === 'string' ? author : undefined);
+    const authorId = author?.id ? String(author.id) : undefined;
+    const views = parseViewsText(n.view_count?.text ?? n.view_count ?? n.views?.text);
+    const isShort = /short|reel/i.test(type) || (Number.isFinite(duration) && duration > 0 && duration <= 70 && /short/i.test(title) === false && /reel|short/i.test(type));
+    const badges = Array.isArray(n.badges) ? n.badges.map((b) => b?.label).filter(Boolean) : [];
+    return {
+      title: String(title),
+      id,
+      url: `https://www.youtube.com/watch?v=${id}`,
+      thumbnail: bestThumb(n.thumbnails),
+      uploaderName: authorName,
+      type: 'video',
+      ...(Number.isFinite(duration) && duration > 0 ? { duration } : {}),
+      ...(views !== undefined ? { views } : {}),
+      ...(n.published?.text ? { publishedAt: String(n.published.text) } : {}),
+      ...(authorId ? { channelId: authorId, channelExternalId: authorId, channelUrl: `https://www.youtube.com/channel/${authorId}` } : {}),
+      ...(authorName ? { channelHandle: String(authorName) } : {}),
+      ...(n.is_live ? { isLive: true } : {}),
+      ...(isShort ? { isShort: true } : {}),
+      ...(badges.length ? { badges } : {}),
+    };
+  } catch { return null; }
+}
+
+/** Mappe un LockupView (renderer moderne : watch-next, search, shelves). */
+export function mapLockupView(n) {
+  try {
+    const ct = String(n.content_type || '').toUpperCase();
+    if (ct && !['VIDEO', 'SHORT', 'MOVIE', 'LIVE'].includes(ct)) return null;
+    const id = n.content_id ? String(n.content_id)
+      : (n.renderer_context?.command_context?.on_tap?.payload?.videoId || null);
+    if (!id) return null;
+    const meta = n.metadata || {};
+    const title = meta.title?.text ?? (typeof meta.title === 'string' ? meta.title : '');
+    if (!title) return null;
+    // Vignette : content_image.image[] (prend la plus large)
+    let thumbnail;
+    try {
+      const imgs = (n.content_image?.image || []).filter((i) => i?.url);
+      thumbnail = imgs.sort((a, b) => (b.width || 0) - (a.width || 0))[0]?.url;
+    } catch {}
+    // Chaîne : 1ère ligne des metadata_rows, sinon label a11y "Go to channel X"
+    let uploaderName;
+    try {
+      const rows = meta.metadata?.metadata_rows || [];
+      const first = rows[0]?.metadata_parts?.[0]?.text;
+      uploaderName = first?.text ?? (typeof first === 'string' ? first : undefined);
+    } catch {}
+    if (!uploaderName) {
+      const a11y = String(meta.image?.a11y_label || '');
+      const m = a11y.match(/^(?:go to channel|aller sur la cha[îi]ne)\s+(.+)$/i);
+      if (m) uploaderName = m[1].trim();
+    }
+    // Vues : 2e ligne ("57K", "1,2 M vues"...), durée : label a11y
+    let views;
+    try {
+      const rows = meta.metadata?.metadata_rows || [];
+      const second = rows[1]?.metadata_parts?.[0]?.text;
+      views = parseViewsText(second?.text ?? second);
+    } catch {}
+    const duration = parseDurationLabel(n.renderer_context?.accessibility_context?.label);
+    return {
+      title: String(title),
+      id,
+      url: `https://www.youtube.com/watch?v=${id}`,
+      thumbnail,
+      uploaderName,
+      type: 'video',
+      ...(duration !== undefined ? { duration } : {}),
+      ...(views !== undefined ? { views } : {}),
+      ...(uploaderName ? { channelHandle: String(uploaderName) } : {}),
+      ...(ct === 'SHORT' ? { isShort: true } : {}),
+      ...(ct === 'LIVE' ? { isLive: true } : {}),
+    };
+  } catch { return null; }
+}
+
+export function mapNodes(nodes) {
+  const seen = new Set();
+  const out = [];
+  for (const n of nodes || []) {
+    const m = mapVideoNode(n);
+    if (!m || seen.has(m.id)) continue;
+    seen.add(m.id);
+    out.push(m);
+  }
+  return out;
+}
+
+// Chaîne de continuations en mémoire : q_hash -> { feed, pages }.
+// (SQLite persiste les pages déjà servies ; la chaîne évite de rejouer les pages 1..N-1.)
+const chainCache = new Map();
+const CHAIN_MAX = 50;
+function chainGet(k) { const h = chainCache.get(k); if (h) { chainCache.delete(k); chainCache.set(k, h); } return h || null; }
+function chainSet(k, v) {
+  if (chainCache.has(k)) chainCache.delete(k);
+  chainCache.set(k, v);
+  while (chainCache.size > CHAIN_MAX) { const o = chainCache.keys().next().value; if (o === undefined) break; chainCache.delete(o); }
+}
+export function innertubeChainStats() { return { chains: chainCache.size, max: CHAIN_MAX }; }
+
+function sortParam(sort) {
+  const s = String(sort || 'relevance').toLowerCase();
+  if (s === 'date') return 'upload_date';
+  if (s === 'views') return 'view_count';
+  return 'relevance';
+}
+
+/**
+ * Recherche InnerTube avec vraie pagination (continuations).
+ * Page 1 ~35-40 vidéos ; pages suivantes via getContinuation() fusionnées (façon SmartTube).
+ */
+export async function searchViaInnerTube(q, opts = {}) {
+  const query = String(q || '').trim();
+  if (query.length < 2) return [];
+  const limit = Math.min(50, Math.max(1, Number(opts?.limit || 24)));
+  const page = Math.min(10, Math.max(1, Number(opts?.page || 1)));
+  const sort = String(opts?.sort || 'relevance');
+  ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
+  const yt = await getSession();
+  const key = `it|${hashSearchKey(`${query.toLowerCase()}|${sort}`)}`;
+  let entry = chainGet(key);
+  let feed = entry?.feed || null;
+  let collected = entry?.items ? [...entry.items] : [];
+  let pagesDone = entry?.pages || 0;
+  // Note : une "page" InnerTube fait ~17-20 vidéos quel que soit `limit`.
+  // On charge donc des continuations jusqu'à couvrir la fenêtre demandée
+  // [start, start+limit[ (façon SmartTube qui remplit son écran au fil des
+  // continuations), au lieu d'aligner 1 page API = 1 page UI.
+  const need = page * limit;
+  try {
+    if (!feed) {
+      feed = await yt.search(query, { sort_by: sortParam(sort) });
+      collected = mapNodes(feed.videos);
+      pagesDone = 1;
+    }
+    let guard = 0;
+    while (collected.length < need && feed?.has_continuation && guard < 12) {
+      guard++;
+      feed = await feed.getContinuation();
+      const fresh = mapNodes(feed.videos).filter((m) => !collected.some((c) => c.id === m.id));
+      collected = collected.concat(fresh);
+      pagesDone++;
+      if (!fresh.length) break;
+    }
+  } catch (e) {
+    if (e?.code === 'yt_innertube_unavailable') throw e;
+    throw Object.assign(new Error(`InnerTube search failed: ${String(e?.message || e).slice(0, 160)}`), { ytStatus: 502, code: 'yt_innertube_failed' });
+  }
+  if (feed) chainSet(key, { feed, items: collected, pages: pagesDone });
+  const start = (page - 1) * limit;
+  return collected.slice(start, start + limit);
+}
+
+/**
+ * Vidéos connexes façon SmartTube (endpoint watch-next) : ce qui alimente
+ * la colonne "À suivre" sous le lecteur.
+ */
+export async function getRelatedViaInnerTube(videoId, limit = 24) {
+  const id = String(videoId || '').trim();
+  if (!id) return [];
+  const n = Math.min(50, Math.max(1, Number(limit || 24)));
+  ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
+  const yt = await getSession();
+  try {
+    const info = await yt.getInfo(id);
+    const feed = info?.watch_next_feed;
+    const arr = Array.isArray(feed) ? feed : (feed ? Array.from(feed) : []);
+    return mapNodes(arr).filter((m) => m.id !== id).slice(0, n);
+  } catch (e) {
+    if (e?.code === 'yt_innertube_unavailable') throw e;
+    throw Object.assign(new Error(`InnerTube related failed: ${String(e?.message || e).slice(0, 160)}`), { ytStatus: 502, code: 'yt_innertube_failed' });
+  }
+}
+
+// -------------------- Transcripts via InnerTube --------------------
+// `getInfo().captions.caption_tracks[]` expose les mêmes URLs timedtext que
+// yt-dlp découvre via un dump complet (`subtitles`/`automatic_captions`).
+// On adapte leur forme vers le format yt-dlp pour réutiliser pickTrack(),
+// orderedTracks() et parseTrackText() de transcript.mjs sans les toucher.
+
+function normCaptionLang(code) {
+  return String(code || '').trim().toLowerCase().replace(/_/g, '-');
+}
+
+/**
+ * Adapte des caption tracks InnerTube brutes vers un pseudo dump yt-dlp
+ * `{ subtitles, automatic_captions }` (kind 'asr' = auto-généré).
+ * Pur et testable offline.
+ */
+export function mapCaptionTracks(captionTracks) {
+  const subtitles = {};
+  const automatic_captions = {};
+  for (const t of captionTracks || []) {
+    const base = String(t?.base_url || '');
+    if (!base) continue;
+    const lang = normCaptionLang(t?.language_code) || 'und';
+    const name = t?.name?.text ?? (typeof t?.name === 'string' ? t.name : lang);
+    const sep = base.includes('?') ? '&' : '?';
+    const track = { url: `${base}${sep}fmt=json3`, ext: 'json3', name: String(name) };
+    const dict = t?.kind === 'asr' ? automatic_captions : subtitles;
+    if (!Array.isArray(dict[lang])) dict[lang] = [];
+    if (!dict[lang].some((x) => x.url === track.url)) dict[lang].push(track);
+  }
+  const languages = Array.from(new Set([...Object.keys(subtitles), ...Object.keys(automatic_captions)]));
+  const trackCount = Object.values(subtitles).concat(Object.values(automatic_captions))
+    .reduce((n, arr) => n + arr.length, 0);
+  return { subtitles, automatic_captions, languages, trackCount };
+}
+
+/**
+ * Découvre les pistes de sous-titres d'une vidéo via InnerTube (0 quota,
+ * pas de spawn yt-dlp). Retourne un pseudo dump yt-dlp prêt pour pickTrack().
+ */
+export async function getCaptionTracksViaInnerTube(videoId) {
+  const id = String(videoId || '').trim();
+  if (!id) return { subtitles: {}, automatic_captions: {}, languages: [], trackCount: 0 };
+  ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
+  const yt = await getSession();
+  try {
+    const info = await yt.getInfo(id);
+    const raw = info?.captions?.caption_tracks || [];
+    return mapCaptionTracks(Array.isArray(raw) ? raw : Array.from(raw));
+  } catch (e) {
+    if (e?.code === 'yt_innertube_unavailable') throw e;
+    throw Object.assign(new Error(`InnerTube captions failed: ${String(e?.message || e).slice(0, 160)}`), { ytStatus: 502, code: 'yt_innertube_failed' });
+  }
+}