feat(youtube): InnerTube-first search, transcripts and watch-next related (Steps 15-18)

- InnerTube layer via pinned youtubei.js 18.1.0 (no quota, no key):
  search with merged continuations (unlimited pages), watch-next
  related with LockupView mapping, caption-track discovery
- 3-layer dispatcher (YT_SEARCH_MODE, default innertube-first):
  innertube -> yt-dlp scrape -> official API, graceful errors.yt
- Robust yt-dlp binary resolution (YT_DLP_PATH > PATH > bundled)
  with systematic API fallback (fixes spawn ENOENT in UI)
- Transcript: InnerTube caption discovery (YT_TRANSCRIPT_SOURCE),
  reusing pickTrack/orderedTracks/parseTrackText; yt-dlp fallback kept
- Watch: sidebar uses real watch-next related[] (/api/details),
  title-search fallback for other providers
- Cache: memory LRU + SQLite (youtube_search_cache, youtube_metrics),
  never persist empty pages; /healthz observability; /api/trending
- Includes pending Step 15/16 leftovers in same files (suggest,
  test scripts); unrelated provider adapters left uncommitted

server/providers/youtube.mjs

@@ -9,6 +9,12 @@
  * @property {string=} uploaderName
  * @property {string=} type
  */
+import {
+  getYouTubeKeys, isKeyFailure, getSearchMode, getScrapeTtlMs,
+  hashSearchKey, ytMetrics,
+} from './youtube-common.mjs';
+import { searchViaScrape } from './youtube-scrape.mjs';
+import { searchViaInnerTube } from './youtube-innertube.mjs';
 
 function parseISODurationToSeconds(iso) {
   if (typeof iso !== 'string' || !iso) return 0;
@@ -20,50 +26,6 @@ function parseISODurationToSeconds(iso) {
   return (hours * 3600) + (minutes * 60) + seconds;
 }
 
-/**
- * Resolve configured YouTube API keys.
- * Accepts YOUTUBE_API_KEYS as JSON array ('["k1","k2"]') or CSV ('k1,k2'),
- * plus the legacy single YOUTUBE_API_KEY as fallback. De-duplicated.
- * @returns {string[]}
- */
-function getYouTubeKeys() {
-  const keys = [];
-  try {
-    const raw = process.env.YOUTUBE_API_KEYS;
-    if (raw && String(raw).trim() && String(raw).trim() !== 'undefined' && String(raw).trim() !== 'null') {
-      const s = String(raw).trim();
-      if (s.startsWith('[')) {
-        try {
-          const arr = JSON.parse(s);
-          if (Array.isArray(arr)) keys.push(...arr.map(v => String(v || '').trim()).filter(Boolean));
-        } catch {}
-      } else {
-        keys.push(...s.split(',').map(v => String(v || '').trim()).filter(Boolean));
-      }
-    }
-  } catch {}
-  try {
-    const single = process.env.YOUTUBE_API_KEY;
-    if (single && String(single).trim()) keys.push(String(single).trim());
-  } catch {}
-  return Array.from(new Set(keys.filter(Boolean)));
-}
-
-/**
- * Classify a YouTube API failure as retryable with another key.
- * - 400 API_KEY_INVALID / "API key expired" -> key is dead, try next.
- * - 403 quotaExceeded / rateLimitExceeded / dailyLimitExceeded -> quota, try next.
- */
-function isKeyFailure(status, data) {
-  try {
-    const reason = data?.error?.errors?.[0]?.reason || '';
-    const message = String(data?.error?.message || '');
-    if (status === 400 && (reason === 'API_KEY_INVALID' || /api key (expired|invalid)/i.test(message))) return true;
-    if (status === 403 && /quota|rateLimit|dailyLimit|userRateLimit/i.test(reason + ' ' + message)) return true;
-  } catch {}
-  return false;
-}
-
 /** GET JSON from YouTube, rotating through keys on key failures. */
 async function ytFetchJson(base, paramsWithoutKey) {
   const keys = getYouTubeKeys();
@@ -76,13 +38,19 @@ async function ytFetchJson(base, paramsWithoutKey) {
   for (const key of keys) {
     const params = new URLSearchParams(paramsWithoutKey);
     params.set('key', key);
+    ytMetrics.apiCalls++;
+    try {
+      const { incYoutubeMetrics } = await import('../db.mjs').catch(() => ({}));
+      if (String(base).includes('/search')) incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 100 });
+      else incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 1 });
+      ytMetrics.quotaUnits += String(base).includes('/search') ? 100 : 1;
+    } catch {}
     const resp = await fetch(`${base}?${params.toString()}`);
     const data = await resp.json().catch(() => ({}));
     if (resp.ok) return data;
     lastStatus = resp.status;
     lastData = data;
     lastError = new Error(`YouTube API error: ${resp.status} ${data?.error?.message || ''}`.trim());
-    // Only rotate to the next key on key-attributable failures; otherwise fail fast.
     if (!isKeyFailure(resp.status, data)) break;
     console.warn(`[YouTube] key ...${String(key).slice(-4)} failed (${resp.status}), trying next key`);
   }
@@ -92,121 +60,278 @@ async function ytFetchJson(base, paramsWithoutKey) {
   throw err;
 }
 
+/**
+ * Parse a `suggestqueries.google.com` (client=youtube) response body into plain strings.
+ * The endpoint answers either bare JSON (`["q",["s1","s2"],...]`) or wrapped in
+ * `window.google.ac.h(...)`. Never throws: unparsable bodies yield [].
+ * @param {string} text
+ * @returns {string[]}
+ */
+export function parseYoutubeSuggestResponse(text) {
+  try {
+    const raw = String(text || '').trim();
+    if (!raw) return [];
+    let payload = raw;
+    const start = raw.indexOf('(');
+    const end = raw.lastIndexOf(')');
+    if (start !== -1 && end > start) payload = raw.slice(start + 1, end);
+    const parsed = JSON.parse(payload);
+    const candidates = Array.isArray(parsed) && Array.isArray(parsed[1]) ? parsed[1] : [];
+    return candidates
+      .map((s) => {
+        if (Array.isArray(s)) s = s[0];
+        return (typeof s === 'string' ? s : String(s ?? '')).trim();
+      })
+      .filter(Boolean)
+      .slice(0, 20);
+  } catch {
+    return [];
+  }
+}
+
+// Mémoire LRU process-wide pour le dispatcher (complète SQLite)
+const memCache = new Map();
+const MEM_MAX = Number(process.env.YT_SCRAPE_MEM_MAX || 300);
+function memGet(k, ttl) {
+  const hit = memCache.get(k);
+  if (!hit) return null;
+  if ((Date.now() - hit.ts) >= ttl) { memCache.delete(k); return null; }
+  memCache.delete(k); memCache.set(k, hit);
+  return hit.items;
+}
+function memSet(k, items) {
+  if (memCache.has(k)) memCache.delete(k);
+  memCache.set(k, { ts: Date.now(), items });
+  while (memCache.size > MEM_MAX) { const o = memCache.keys().next().value; if (o === undefined) break; memCache.delete(o); }
+}
+export function ytScrapeCacheStats() {
+  return { memEntries: memCache.size, memMax: MEM_MAX };
+}
+
+async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance' } = {}) {
+  const keys = getYouTubeKeys();
+  if (!keys.length) {
+    throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' });
+  }
+  let order = 'relevance';
+  if (sort === 'date') order = 'date';
+  else if (sort === 'views') order = 'viewCount';
+  const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
+  const targetPage = Math.max(1, Number(page || 1));
+  let pageToken = '';
+  let currentPage = 1;
+  let lastItems = [];
+  while (currentPage <= targetPage) {
+    const params = {
+      part: 'snippet', q, type: 'video', maxResults: String(perPage), order,
+      videoEmbeddable: 'true', safeSearch: 'moderate',
+    };
+    if (pageToken) params.pageToken = pageToken;
+    const data = await ytFetchJson('https://www.googleapis.com/youtube/v3/search', params);
+    if (currentPage === targetPage) { lastItems = Array.isArray(data.items) ? data.items : []; break; }
+    const next = data.nextPageToken;
+    if (!next) { lastItems = []; break; }
+    pageToken = String(next);
+    currentPage++;
+  }
+  const videoIds = (lastItems || []).map((item) => item?.id?.videoId).filter(Boolean);
+  const detailsMap = new Map();
+  if (videoIds.length > 0) {
+    try {
+      const detailsData = await ytFetchJson('https://www.googleapis.com/youtube/v3/videos', {
+        part: 'contentDetails,statistics,status', id: videoIds.join(','),
+      });
+      for (const vid of detailsData?.items || []) if (vid?.id) detailsMap.set(vid.id, vid);
+    } catch (e) {
+      console.warn('[YouTube] details fetch failed, continuing without durations:', e?.message || e);
+    }
+  }
+  return (lastItems || []).map((item) => {
+    const videoId = item?.id?.videoId;
+    const snippet = item?.snippet || {};
+    const thumb = snippet.thumbnails?.high?.url || snippet.thumbnails?.medium?.url || snippet.thumbnails?.default?.url || undefined;
+    const details = videoId ? detailsMap.get(videoId) : null;
+    const duration = parseISODurationToSeconds(details?.contentDetails?.duration || '');
+    const views = details?.statistics?.viewCount != null ? Number(details.statistics.viewCount) : undefined;
+    const channelId = snippet.channelId || undefined;
+    const embeddable = details?.status ? details.status.embeddable !== false : undefined;
+    return {
+      title: snippet.title || '', id: videoId,
+      url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined,
+      thumbnail: thumb, uploaderName: snippet.channelTitle || undefined, type: 'video',
+      duration: duration > 0 ? duration : undefined, views,
+      publishedAt: snippet.publishedAt || undefined, channelId,
+      channelHandle: snippet.channelTitle || undefined, channelExternalId: channelId,
+      channelUrl: channelId ? `https://www.youtube.com/channel/${channelId}` : undefined, embeddable,
+    };
+  });
+}
+
 /** @type {{ id: 'yt', label: string, search: (q: string, opts: { limit: number, page?: number, sort?: 'relevance'|'date'|'views' }) => Promise<Suggestion[]> }} */
 const handler = {
   id: 'yt',
   label: 'YouTube',
   async search(q, opts) {
     const { limit = 10, page = 1, sort = 'relevance' } = opts || {};
+    const mode = getSearchMode();
+    const ttl = getScrapeTtlMs();
+    const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
+    const key = `yt|${hashSearchKey(`${String(q).toLowerCase().trim()}|${perPage}|${page}|${sort}|${mode}`)}`;
+    // 1) mémoire
+    const memHit = memGet(key, ttl);
+    if (memHit) { ytMetrics.scrapeHits++; return memHit; }
+    // 2) SQLite
     try {
-      const keys = getYouTubeKeys();
-      if (!keys.length) {
-        throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' });
+      const { getCachedYoutubeSearch } = await import('../db.mjs');
+      const cached = getCachedYoutubeSearch(key);
+      if (cached?.items) {
+        ytMetrics.scrapeHits++;
+        memSet(key, cached.items);
+        return cached.items;
       }
-
-      let order = 'relevance';
-      if (sort === 'date') order = 'date';
-      else if (sort === 'views') order = 'viewCount';
-
-      // Iterate nextPageToken to reach the requested page (1-based)
-      const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
-      const targetPage = Math.max(1, Number(page || 1));
-      let pageToken = '';
-      let currentPage = 1;
-      let lastItems = [];
-
-      while (currentPage <= targetPage) {
-        const params = {
-          part: 'snippet',
-          q: q,
-          type: 'video',
-          maxResults: String(perPage),
-          order,
-          // Ne retourner que des vidéos lisibles en embed (évite l'erreur 153 côté player).
-          // Les vidéos non-embeddables sont filtrées via le champ status ci-dessous pour /videos.
-          videoEmbeddable: 'true',
-          safeSearch: 'moderate',
-        };
-        if (pageToken) params.pageToken = pageToken;
-
-        const data = await ytFetchJson('https://www.googleapis.com/youtube/v3/search', params);
-
-        if (currentPage === targetPage) {
-          lastItems = Array.isArray(data.items) ? data.items : [];
-          break;
-        }
-
-        // Prepare for next iteration
-        const next = data.nextPageToken;
-        if (!next) {
-          // No more pages; requested page beyond available results
-          lastItems = [];
-          break;
-        }
-        pageToken = String(next);
-        currentPage++;
+    } catch {}
+    const persist = async (items, source) => {
+      // Ne jamais persister un résultat vide : une page vide transitoire
+      // (continuation expirée, raté réseau partiel) ne doit pas empoisonner
+      // le cache et bloquer les requêtes suivantes.
+      if (!Array.isArray(items) || items.length === 0) return items;
+      memSet(key, items);
+      try {
+        const { setCachedYoutubeSearch, incYoutubeMetrics } = await import('../db.mjs');
+        setCachedYoutubeSearch(key, q, items, source, ttl);
+        if (source === 'scrape') incYoutubeMetrics({ scrapeCalls: 1 });
+      } catch {}
+      return items;
+    };
+    const t0 = Date.now();
+    const tryScrape = async () => {
+      ytMetrics.scrapeCalls++;
+      try { const { incYoutubeMetrics } = await import('../db.mjs'); incYoutubeMetrics({ scrapeCalls: 1 }); } catch {}
+      return searchViaScrape(q, { limit: perPage, page, sort });
+    };
+    const tryApi = () => searchViaApi(q, { limit: perPage, page, sort });
+    const tryInnerTube = () => searchViaInnerTube(q, { limit: perPage, page, sort });
+    const log = (source, extra = '') => console.log(`[YT search] source=${source} mode=${mode} latency=${Date.now() - t0}ms results=${extra}`);
+    try {
+      if (mode === 'api-only') {
+        const items = await tryApi();
+        log('api', items.length);
+        return persist(items, 'api');
       }
-
-      const videoIds = (lastItems || [])
-        .map(item => item?.id?.videoId)
-        .filter(Boolean);
-
-      const detailsMap = new Map();
-      if (videoIds.length > 0) {
+      if (mode === 'scrape-only') {
+        const items = await tryScrape();
+        log('scrape', items.length);
+        return persist(items, 'scrape');
+      }
+      if (mode === 'innertube-only') {
+        const items = await tryInnerTube();
+        log('innertube', items.length);
+        return persist(items, 'innertube');
+      }
+      if (mode === 'api-first') {
         try {
-          const detailsData = await ytFetchJson('https://www.googleapis.com/youtube/v3/videos', {
-            part: 'contentDetails,statistics,status',
-            id: videoIds.join(','),
-          });
-          for (const vid of detailsData?.items || []) {
-            if (vid?.id) detailsMap.set(vid.id, vid);
-          }
-        } catch (e) {
-          console.warn('[YouTube] details fetch failed, continuing without durations:', e?.message || e);
+          const items = await tryApi();
+          log('api', items.length);
+          return persist(items, 'api');
+        } catch (apiErr) {
+          ytMetrics.fallbacks++;
+          console.warn(`[YT search] api-first fallback to scrape: ${apiErr?.message || apiErr}`);
+          const items = await tryScrape();
+          log('scrape(fallback)', items.length);
+          return persist(items, 'scrape');
         }
       }
-
-      return (lastItems || []).map(item => {
-        const videoId = item?.id?.videoId;
-        const snippet = item?.snippet || {};
-        const thumb = snippet.thumbnails?.high?.url
-          || snippet.thumbnails?.medium?.url
-          || snippet.thumbnails?.default?.url
-          || undefined;
-
-        const details = videoId ? detailsMap.get(videoId) : null;
-        const isoDuration = details?.contentDetails?.duration || '';
-        const duration = parseISODurationToSeconds(isoDuration);
-        const views = details?.statistics?.viewCount != null ? Number(details.statistics.viewCount) : undefined;
-        const channelId = snippet.channelId || undefined;
-        const channelHandle = snippet.channelTitle || undefined;
-        // status.embeddable === false -> le player renvoie l'erreur 153 ("Video configuration error").
-        const embeddable = details?.status ? details.status.embeddable !== false : undefined;
-
-        return {
-          title: snippet.title || '',
-          id: videoId,
-          url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined,
-          thumbnail: thumb,
-          uploaderName: snippet.channelTitle || undefined,
-          type: 'video',
-          duration: duration > 0 ? duration : undefined,
-          views,
-          publishedAt: snippet.publishedAt || undefined,
-          channelId,
-          channelHandle,
-          channelExternalId: channelId,
-          channelUrl: channelId ? `https://www.youtube.com/channel/${channelId}` : undefined,
-          embeddable,
-        };
-      });
+      if (mode === 'scrape-first') {
+        // scrape -> API (comportement historique, sans InnerTube)
+        try {
+          const items = await tryScrape();
+          log('scrape', items.length);
+          return persist(items, 'scrape');
+        } catch (scrapeErr) {
+          ytMetrics.scrapeErrors++;
+          ytMetrics.fallbacks++;
+          console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`);
+          try {
+            const items = await tryApi();
+            log('api(fallback)', items.length);
+            return persist(items, 'api');
+          } catch (apiErr) {
+            const noBinary = scrapeErr?.code === 'yt_scrape_no_binary';
+            const noKey = apiErr?.code === 'youtube_api_key_unavailable';
+            if (noBinary && noKey) {
+              const err = new Error('YouTube indisponible : yt-dlp introuvable (définissez YT_DLP_PATH) et aucune clé YOUTUBE_API_KEY configurée.');
+              err.ytStatus = 503; err.code = 'youtube_no_source';
+              throw err;
+            }
+            console.error(`[YT search] both sources failed. scrape=${scrapeErr?.message} api=${apiErr?.message}`);
+            throw scrapeErr;
+          }
+        }
+      }
+      // innertube-first (défaut) : InnerTube -> scrape yt-dlp -> API officielle.
+      // Chaque couche ne fait jamais échouer la recherche à elle seule.
+      const errors = [];
+      try {
+        const items = await tryInnerTube();
+        log('innertube', items.length);
+        return persist(items, 'innertube');
+      } catch (itErr) {
+        ytMetrics.innertubeErrors = (ytMetrics.innertubeErrors || 0) + 1;
+        ytMetrics.fallbacks++;
+        errors.push(`innertube=${itErr?.message}`);
+        console.warn(`[YT search] innertube failed (${itErr?.code || 'unknown'}), fallback to scrape`);
+      }
+      try {
+        const items = await tryScrape();
+        log('scrape(fallback)', items.length);
+        return persist(items, 'scrape');
+      } catch (scrapeErr) {
+        ytMetrics.scrapeErrors++;
+        ytMetrics.fallbacks++;
+        errors.push(`scrape=${scrapeErr?.message}`);
+        console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`);
+      }
+      try {
+        const items = await tryApi();
+        log('api(fallback)', items.length);
+        return persist(items, 'api');
+      } catch (apiErr) {
+        errors.push(`api=${apiErr?.message}`);
+        const noSource = !getYouTubeKeys().length;
+        if (noSource) {
+          const err = new Error(`YouTube indisponible (${errors.join(' | ').slice(0, 220)}). Vérifiez le réseau/YT_DLP_PATH ou ajoutez YOUTUBE_API_KEYS.`);
+          err.ytStatus = 503; err.code = 'youtube_no_source';
+          throw err;
+        }
+        console.error(`[YT search] all sources failed. ${errors.join(' | ')}`);
+        const err = new Error(errors[0] || 'search_failed');
+        err.ytStatus = 502; err.code = 'youtube_all_sources_failed';
+        throw err;
+      }
     } catch (error) {
-      // Log explicite (clé expirée / quota) puis remonte l'erreur pour que
-      // /api/search la reporte dans `errors.yt` au lieu d'un groupe vide silencieux.
       const status = error?.ytStatus || 500;
       console.error(`YouTube search error (status ${status}):`, error?.message || error);
       throw error;
     }
+  },
+  async suggest(q, opts) {
+    const limit = Math.min(20, Math.max(1, Number(opts?.limit || 10)));
+    const query = String(q || '').trim();
+    if (query.length < 2) return [];
+    try {
+      const params = new URLSearchParams({ client: 'youtube', ds: 'yt', hl: 'fr', q: query });
+      const headers = { 'User-Agent': 'Mozilla/5.0' };
+      const proxy = String(process.env.YT_EGRESS_PROXY || '').trim();
+      const fetchOpts = { headers, signal: AbortSignal.timeout(6000) };
+      // Note : suggest reste sans proxy (endpoint Google public léger) sauf si egress configuré côté infra.
+      if (proxy) console.debug?.('[YT suggest] egress proxy configured, direct fetch kept for suggest');
+      const resp = await fetch(`https://suggestqueries.google.com/complete/search?${params.toString()}`, fetchOpts);
+      if (!resp.ok) return [];
+      const text = await resp.text();
+      return parseYoutubeSuggestResponse(text).slice(0, limit);
+    } catch {
+      return [];
+    }
   }
 };