/** * @typedef {Object} Suggestion * @property {string} title * @property {string} id * @property {number=} duration * @property {boolean=} isShort * @property {string=} url * @property {string=} thumbnail * @property {string=} uploaderName * @property {string=} type */ import { getYouTubeKeys, isKeyFailure, getEffectiveSearchMode, getScrapeTtlMs, hashSearchKey, ytMetrics, } from './youtube-common.mjs'; import { searchViaScrape } from './youtube-scrape.mjs'; import { searchViaInnerTube } from './youtube-innertube.mjs'; import { apiSearchParams, filtersCacheKey } from '../search-filters.mjs'; function parseISODurationToSeconds(iso) { if (typeof iso !== 'string' || !iso) return 0; const match = iso.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/); if (!match) return 0; const hours = Number(match[1] || 0); const minutes = Number(match[2] || 0); const seconds = Number(match[3] || 0); return (hours * 3600) + (minutes * 60) + seconds; } /** GET JSON from YouTube, rotating through keys on key failures. */ async function ytFetchJson(base, paramsWithoutKey) { const keys = getYouTubeKeys(); if (!keys.length) { throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' }); } let lastError = null; let lastStatus = 500; let lastData = null; for (const key of keys) { const params = new URLSearchParams(paramsWithoutKey); params.set('key', key); ytMetrics.apiCalls++; try { const { incYoutubeMetrics } = await import('../db.mjs').catch(() => ({})); if (String(base).includes('/search')) incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 100 }); else incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 1 }); ytMetrics.quotaUnits += String(base).includes('/search') ? 100 : 1; } catch {} const resp = await fetch(`${base}?${params.toString()}`); const data = await resp.json().catch(() => ({})); if (resp.ok) return data; lastStatus = resp.status; lastData = data; lastError = new Error(`YouTube API error: ${resp.status} ${data?.error?.message || ''}`.trim()); if (!isKeyFailure(resp.status, data)) break; console.warn(`[YouTube] key ...${String(key).slice(-4)} failed (${resp.status}), trying next key`); } const err = lastError || new Error(`YouTube API error: ${lastStatus}`); err.ytStatus = lastStatus; err.ytData = lastData; throw err; } /** * Parse a `suggestqueries.google.com` (client=youtube) response body into plain strings. * The endpoint answers either bare JSON (`["q",["s1","s2"],...]`) or wrapped in * `window.google.ac.h(...)`. Never throws: unparsable bodies yield []. * @param {string} text * @returns {string[]} */ export function parseYoutubeSuggestResponse(text) { try { const raw = String(text || '').trim(); if (!raw) return []; let payload = raw; const start = raw.indexOf('('); const end = raw.lastIndexOf(')'); if (start !== -1 && end > start) payload = raw.slice(start + 1, end); const parsed = JSON.parse(payload); const candidates = Array.isArray(parsed) && Array.isArray(parsed[1]) ? parsed[1] : []; return candidates .map((s) => { if (Array.isArray(s)) s = s[0]; return (typeof s === 'string' ? s : String(s ?? '')).trim(); }) .filter(Boolean) .slice(0, 20); } catch { return []; } } // Mémoire LRU process-wide pour le dispatcher (complète SQLite) const memCache = new Map(); const MEM_MAX = Number(process.env.YT_SCRAPE_MEM_MAX || 300); function memGet(k, ttl) { const hit = memCache.get(k); if (!hit) return null; if ((Date.now() - hit.ts) >= ttl) { memCache.delete(k); return null; } memCache.delete(k); memCache.set(k, hit); return hit.items; } function memSet(k, items) { if (memCache.has(k)) memCache.delete(k); memCache.set(k, { ts: Date.now(), items }); while (memCache.size > MEM_MAX) { const o = memCache.keys().next().value; if (o === undefined) break; memCache.delete(o); } } export function ytScrapeCacheStats() { return { memEntries: memCache.size, memMax: MEM_MAX }; } /** * Phase 3.6 — cache des `pageToken` YouTube. * Avant : atteindre la page N exigeait de rejouer les pages 1..N à chaque visite * (`while (currentPage <= targetPage)`), soit jusqu'à 100 unités de quota pour un * simple « page 5 ». La Data API est paginée par jeton, pas par offset : on mémorise * donc la chaîne de jetons et chaque page thereafter coûte 1 appel. * TTL 15 min, borné à 200 entrées par instance. * @type {Map} */ const PAGE_TOKEN_TTL_MS = 15 * 60 * 1000; const pageTokenCache = new Map(); function pageTokenCacheKey(q, order, perPage, extra) { return `ytpage|${hashSearchKey(`${String(q).toLowerCase().trim()}|${order}|${perPage}|${JSON.stringify(extra || {})}`)}`; } export function clearYouTubePageTokenCache() { pageTokenCache.clear(); } async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance', filters = null } = {}) { const keys = getYouTubeKeys(); if (!keys.length) { throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' }); } let order = 'relevance'; if (sort === 'date') order = 'date'; else if (sort === 'views') order = 'viewCount'; // Filtres natifs de l'API v3 : type, videoDuration, publishedAfter. const extra = apiSearchParams(filters); const perPage = Math.min(Math.max(1, Number(limit || 10)), 50); const targetPage = Math.max(1, Number(page || 1)); const cacheKey = pageTokenCacheKey(q, order, perPage, extra); const hit = pageTokenCache.get(cacheKey); const entry = (hit && (Date.now() - hit.ts) < PAGE_TOKEN_TTL_MS) ? hit : null; if (!entry && hit) pageTokenCache.delete(cacheKey); if (pageTokenCache.size >= 200) { for (const [k, v] of pageTokenCache) { if ((Date.now() - v.ts) >= PAGE_TOKEN_TTL_MS) pageTokenCache.delete(k); if (pageTokenCache.size < 150) break; } } /** `tokens[i]` = pageToken permettant de fetch la page i+2. Doit être contigu depuis 0. */ const tokens = entry ? entry.tokens : []; const buildParams = (pageToken) => ({ part: 'snippet', q, maxResults: String(perPage), order, videoEmbeddable: 'true', safeSearch: 'moderate', ...extra, ...(pageToken ? { pageToken } : {}), }); // Si le jeton de la page cible est déjà connu, on saute directement dessus : // 1 appel au lieu de N. Sinon on rejoue les pages manquantes à partir d'où // la chaîne s'arrête (jamais depuis le début si le cache est partiel). const startPage = Math.min(targetPage, tokens.length + 1); let pageToken = startPage >= 2 ? (tokens[startPage - 2] || '') : ''; let lastItems = []; for (let currentPage = startPage; currentPage <= targetPage; currentPage++) { const data = await ytFetchJson('https://www.googleapis.com/youtube/v3/search', buildParams(pageToken)); if (currentPage === targetPage) { lastItems = Array.isArray(data.items) ? data.items : []; break; } const next = data.nextPageToken; if (!next) { lastItems = []; break; } tokens[currentPage - 1] = String(next); pageToken = String(next); } pageTokenCache.set(cacheKey, { ts: Date.now(), tokens }); const videoIds = (lastItems || []).map((item) => item?.id?.videoId).filter(Boolean); const detailsMap = new Map(); if (videoIds.length > 0) { try { const detailsData = await ytFetchJson('https://www.googleapis.com/youtube/v3/videos', { part: 'contentDetails,statistics,status', id: videoIds.join(','), }); for (const vid of detailsData?.items || []) if (vid?.id) detailsMap.set(vid.id, vid); } catch (e) { console.warn('[YouTube] details fetch failed, continuing without durations:', e?.message || e); } } // Phase 1.7 - `search.list` ne renvoie QUE la vignette de la video, jamais // celle de la chaine. Un seul appel `channels.list` par page suffit ahydrater // `uploaderAvatar` pour tous les resultats. Degradation silencieuse : si aucune // cle n'est disponible ou si l'appel echoue, on rend la main sans avatar // (jamais d'image inventee). const channelAvatars = new Map(); try { const channelIds = Array.from(new Set( (lastItems || []) .map((i) => i?.snippet?.channelId || i?.id?.channelId) .filter(Boolean), )).slice(0, 50); if (channelIds.length > 0) { const chData = await ytFetchJson('https://www.googleapis.com/youtube/v3/channels', { part: 'snippet', id: channelIds.join(','), }); for (const ch of chData?.items || []) { const t = ch?.snippet?.thumbnails; const url = t?.high?.url || t?.medium?.url || t?.default?.url; if (ch?.id && url) channelAvatars.set(ch.id, url); } } } catch (e) { console.warn('[YouTube] channel avatars fetch failed, continuing without:', e?.message || e); } return (lastItems || []).map((item) => { const videoId = item?.id?.videoId; // Une recherche de type `channel` renvoie des channelId, pas des videoId. const rawId = videoId || item?.id?.channelId || null; const isChannel = !videoId && !!item?.id?.channelId; if (!rawId) return null; const snippet = item?.snippet || {}; const thumb = snippet.thumbnails?.high?.url || snippet.thumbnails?.medium?.url || snippet.thumbnails?.default?.url || undefined; const details = videoId ? detailsMap.get(videoId) : null; const duration = parseISODurationToSeconds(details?.contentDetails?.duration || ''); const views = details?.statistics?.viewCount != null ? Number(details.statistics.viewCount) : undefined; const channelId = snippet.channelId || item?.id?.channelId || undefined; const embeddable = details?.status ? details.status.embeddable !== false : undefined; return { title: snippet.title || '', id: rawId, url: isChannel ? `https://www.youtube.com/channel/${rawId}` : `https://www.youtube.com/watch?v=${rawId}`, thumbnail: thumb, uploaderName: snippet.channelTitle || undefined, // Phase 1.7 - avatar de la chaine (absent = pas d'avatar, pas de repli casse). ...(channelId && channelAvatars.get(channelId) ? { uploaderAvatar: channelAvatars.get(channelId), channelAvatarUrl: channelAvatars.get(channelId) } : {}), type: isChannel ? 'channel' : 'video', duration: duration > 0 ? duration : undefined, views, // Phase 1.7bis - `likeCount` est gratuit sur le meme appel // `videos?part=statistics` que la duree : autant le remonter pendant qu'on y est. ...(details?.statistics?.likeCount != null ? { likes: Number(details.statistics.likeCount) } : {}), // `liveBroadcastContent` vient du snippet de la recherche (live/upcoming/none). ...(snippet?.liveBroadcastContent === 'live' ? { isLive: true, type: 'live' } : {}), ...(snippet?.liveBroadcastContent === 'upcoming' && !isChannel ? { type: 'upcoming' } : {}), publishedAt: snippet.publishedAt || undefined, channelId, channelHandle: snippet.channelTitle || undefined, channelExternalId: channelId, channelUrl: channelId ? `https://www.youtube.com/channel/${channelId}` : undefined, embeddable, }; }).filter(Boolean); } /** * @typedef {Object} SearchFilters * @property {string} [type] all|video|shorts|live|channel * @property {string} [duration] all|short|medium|long * @property {string} [period] all|hour|today|week|month|year * @property {string} [sort] relevance|date|views */ /** @type {{ id: 'yt', label: string, search: (q: string, opts: { limit: number, page?: number, sort?: 'relevance'|'date'|'views', filters?: SearchFilters }) => Promise }} */ const handler = { id: 'yt', label: 'YouTube', async search(q, opts) { const { limit = 10, page = 1, sort = 'relevance', filters = null } = opts || {}; // Phase 4.4 : on branche sur le mode **effectif** (eventuellement force sur // scrape-first par la bascule automatique), pas sur l'intention. La cle de // cache inclut ce mode : sinon une bascule ferait servir des resultats // InnerTube comme s'ils venaient du scrape. const mode = getEffectiveSearchMode(); const ttl = getScrapeTtlMs(); const perPage = Math.min(Math.max(1, Number(limit || 10)), 50); // La signature des filtres entre dans la clé de cache : deux recherches // identiques avec des filtres différents ne doivent pas se partager de cache. const key = `yt|${hashSearchKey(`${String(q).toLowerCase().trim()}|${perPage}|${page}|${sort}|${mode}|${filtersCacheKey(filters)}`)}`; // 1) mémoire const memHit = memGet(key, ttl); if (memHit) { ytMetrics.scrapeHits++; return memHit; } // 2) SQLite try { const { getCachedYoutubeSearch } = await import('../db.mjs'); const cached = getCachedYoutubeSearch(key); if (cached?.items) { ytMetrics.scrapeHits++; memSet(key, cached.items); return cached.items; } } catch {} const persist = async (items, source) => { // Ne jamais persister un résultat vide : une page vide transitoire // (continuation expirée, raté réseau partiel) ne doit pas empoisonner // le cache et bloquer les requêtes suivantes. if (!Array.isArray(items) || items.length === 0) return items; memSet(key, items); try { const { setCachedYoutubeSearch, incYoutubeMetrics } = await import('../db.mjs'); setCachedYoutubeSearch(key, q, items, source, ttl); if (source === 'scrape') incYoutubeMetrics({ scrapeCalls: 1 }); } catch {} return items; }; const t0 = Date.now(); /** * Phase 4.4 — alimente la bascule automatique. * * Deux choses indispensables, et la premiere etait absente : * 1. l'appel InnerTube est un VRAI appel sortant, il doit etre mesure dans * `provider_metrics` sous le provider `yt` ; * 2. le snapshot est lu APRES cet increment, sinon la tentative courante * n'est pas comptee. * Sans (1), la ligne `yt` restait absente de `provider_metrics` (le registre * met en cache les autres fournisseurs, pas YouTube), donc * `recordInnerTubeOutcome` voyait toujours `calls === 0`, la condition * `row.calls < 3` sortait toujours, et le failover ne pouvait jamais * se declencher en production — uniquement dans les tests, qui injectaient * un snapshot synthetique. */ const noteInnerTube = async (ok, latencyMs, error) => { try { const [{ providerMetricsSnapshot, incProviderMetrics }, common] = await Promise.all([ import('../db.mjs'), import('./youtube-common.mjs'), ]); incProviderMetrics('yt', { ok, latencyMs, error: error || null }); common.recordInnerTubeOutcome(ok, providerMetricsSnapshot({ hours: 1 })); } catch {} }; const tryScrape = async () => { ytMetrics.scrapeCalls++; try { const { incYoutubeMetrics } = await import('../db.mjs'); incYoutubeMetrics({ scrapeCalls: 1 }); } catch {} return searchViaScrape(q, { limit: perPage, page, sort, filters }); }; const tryApi = () => searchViaApi(q, { limit: perPage, page, sort, filters }); const tryInnerTube = () => searchViaInnerTube(q, { limit: perPage, page, sort, filters }); const log = (source, extra = '') => console.log(`[YT search] source=${source} mode=${mode} latency=${Date.now() - t0}ms results=${extra}`); try { if (mode === 'api-only') { const items = await tryApi(); log('api', items.length); return persist(items, 'api'); } if (mode === 'scrape-only') { const items = await tryScrape(); log('scrape', items.length); return persist(items, 'scrape'); } if (mode === 'innertube-only') { const items = await tryInnerTube(); log('innertube', items.length); return persist(items, 'innertube'); } if (mode === 'api-first') { try { const items = await tryApi(); log('api', items.length); return persist(items, 'api'); } catch (apiErr) { ytMetrics.fallbacks++; console.warn(`[YT search] api-first fallback to scrape: ${apiErr?.message || apiErr}`); const items = await tryScrape(); log('scrape(fallback)', items.length); return persist(items, 'scrape'); } } if (mode === 'scrape-first') { // scrape -> API (comportement historique, sans InnerTube) try { const items = await tryScrape(); log('scrape', items.length); return persist(items, 'scrape'); } catch (scrapeErr) { ytMetrics.scrapeErrors++; ytMetrics.fallbacks++; console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`); try { const items = await tryApi(); log('api(fallback)', items.length); return persist(items, 'api'); } catch (apiErr) { const noBinary = scrapeErr?.code === 'yt_scrape_no_binary'; const noKey = apiErr?.code === 'youtube_api_key_unavailable'; if (noBinary && noKey) { const err = new Error('YouTube indisponible : yt-dlp introuvable (définissez YT_DLP_PATH) et aucune clé YOUTUBE_API_KEY configurée.'); err.ytStatus = 503; err.code = 'youtube_no_source'; throw err; } console.error(`[YT search] both sources failed. scrape=${scrapeErr?.message} api=${apiErr?.message}`); throw scrapeErr; } } } // innertube-first (défaut) : InnerTube -> scrape yt-dlp -> API officielle. // Chaque couche ne fait jamais échouer la recherche à elle seule. const errors = []; try { const tIt = Date.now(); const items = await tryInnerTube(); log('innertube', items.length); await noteInnerTube(true, Date.now() - tIt); return persist(items, 'innertube'); } catch (itErr) { ytMetrics.innertubeErrors = (ytMetrics.innertubeErrors || 0) + 1; ytMetrics.fallbacks++; errors.push(`innertube=${itErr?.message}`); console.warn(`[YT search] innertube failed (${itErr?.code || 'unknown'}), fallback to scrape`); await noteInnerTube(false, Date.now() - tIt, itErr?.code || itErr?.message); } try { const items = await tryScrape(); log('scrape(fallback)', items.length); return persist(items, 'scrape'); } catch (scrapeErr) { ytMetrics.scrapeErrors++; ytMetrics.fallbacks++; errors.push(`scrape=${scrapeErr?.message}`); console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`); } try { const items = await tryApi(); log('api(fallback)', items.length); return persist(items, 'api'); } catch (apiErr) { errors.push(`api=${apiErr?.message}`); const noSource = !getYouTubeKeys().length; if (noSource) { const err = new Error(`YouTube indisponible (${errors.join(' | ').slice(0, 220)}). Vérifiez le réseau/YT_DLP_PATH ou ajoutez YOUTUBE_API_KEYS.`); err.ytStatus = 503; err.code = 'youtube_no_source'; throw err; } console.error(`[YT search] all sources failed. ${errors.join(' | ')}`); const err = new Error(errors[0] || 'search_failed'); err.ytStatus = 502; err.code = 'youtube_all_sources_failed'; throw err; } } catch (error) { const status = error?.ytStatus || 500; console.error(`YouTube search error (status ${status}):`, error?.message || error); throw error; } }, async suggest(q, opts) { const limit = Math.min(20, Math.max(1, Number(opts?.limit || 10))); const query = String(q || '').trim(); if (query.length < 2) return []; try { const params = new URLSearchParams({ client: 'youtube', ds: 'yt', hl: 'fr', q: query }); const headers = { 'User-Agent': 'Mozilla/5.0' }; const proxy = String(process.env.YT_EGRESS_PROXY || '').trim(); const fetchOpts = { headers, signal: AbortSignal.timeout(6000) }; // Note : suggest reste sans proxy (endpoint Google public léger) sauf si egress configuré côté infra. if (proxy) console.debug?.('[YT suggest] egress proxy configured, direct fetch kept for suggest'); const resp = await fetch(`https://suggestqueries.google.com/complete/search?${params.toString()}`, fetchOpts); if (!resp.ok) return []; const text = await resp.text(); return parseYoutubeSuggestResponse(text).slice(0, limit); } catch { return []; } } }; export default handler;