/** * @typedef {Object} Suggestion * @property {string} title * @property {string} id * @property {number=} duration * @property {boolean=} isShort * @property {string=} url * @property {string=} thumbnail * @property {string=} uploaderName * @property {string=} type */ import { getYouTubeKeys, isKeyFailure, getSearchMode, getScrapeTtlMs, hashSearchKey, ytMetrics, } from './youtube-common.mjs'; import { searchViaScrape } from './youtube-scrape.mjs'; import { searchViaInnerTube } from './youtube-innertube.mjs'; function parseISODurationToSeconds(iso) { if (typeof iso !== 'string' || !iso) return 0; const match = iso.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/); if (!match) return 0; const hours = Number(match[1] || 0); const minutes = Number(match[2] || 0); const seconds = Number(match[3] || 0); return (hours * 3600) + (minutes * 60) + seconds; } /** GET JSON from YouTube, rotating through keys on key failures. */ async function ytFetchJson(base, paramsWithoutKey) { const keys = getYouTubeKeys(); if (!keys.length) { throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' }); } let lastError = null; let lastStatus = 500; let lastData = null; for (const key of keys) { const params = new URLSearchParams(paramsWithoutKey); params.set('key', key); ytMetrics.apiCalls++; try { const { incYoutubeMetrics } = await import('../db.mjs').catch(() => ({})); if (String(base).includes('/search')) incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 100 }); else incYoutubeMetrics?.({ apiCalls: 1, quotaUnits: 1 }); ytMetrics.quotaUnits += String(base).includes('/search') ? 100 : 1; } catch {} const resp = await fetch(`${base}?${params.toString()}`); const data = await resp.json().catch(() => ({})); if (resp.ok) return data; lastStatus = resp.status; lastData = data; lastError = new Error(`YouTube API error: ${resp.status} ${data?.error?.message || ''}`.trim()); if (!isKeyFailure(resp.status, data)) break; console.warn(`[YouTube] key ...${String(key).slice(-4)} failed (${resp.status}), trying next key`); } const err = lastError || new Error(`YouTube API error: ${lastStatus}`); err.ytStatus = lastStatus; err.ytData = lastData; throw err; } /** * Parse a `suggestqueries.google.com` (client=youtube) response body into plain strings. * The endpoint answers either bare JSON (`["q",["s1","s2"],...]`) or wrapped in * `window.google.ac.h(...)`. Never throws: unparsable bodies yield []. * @param {string} text * @returns {string[]} */ export function parseYoutubeSuggestResponse(text) { try { const raw = String(text || '').trim(); if (!raw) return []; let payload = raw; const start = raw.indexOf('('); const end = raw.lastIndexOf(')'); if (start !== -1 && end > start) payload = raw.slice(start + 1, end); const parsed = JSON.parse(payload); const candidates = Array.isArray(parsed) && Array.isArray(parsed[1]) ? parsed[1] : []; return candidates .map((s) => { if (Array.isArray(s)) s = s[0]; return (typeof s === 'string' ? s : String(s ?? '')).trim(); }) .filter(Boolean) .slice(0, 20); } catch { return []; } } // Mémoire LRU process-wide pour le dispatcher (complète SQLite) const memCache = new Map(); const MEM_MAX = Number(process.env.YT_SCRAPE_MEM_MAX || 300); function memGet(k, ttl) { const hit = memCache.get(k); if (!hit) return null; if ((Date.now() - hit.ts) >= ttl) { memCache.delete(k); return null; } memCache.delete(k); memCache.set(k, hit); return hit.items; } function memSet(k, items) { if (memCache.has(k)) memCache.delete(k); memCache.set(k, { ts: Date.now(), items }); while (memCache.size > MEM_MAX) { const o = memCache.keys().next().value; if (o === undefined) break; memCache.delete(o); } } export function ytScrapeCacheStats() { return { memEntries: memCache.size, memMax: MEM_MAX }; } async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance' } = {}) { const keys = getYouTubeKeys(); if (!keys.length) { throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' }); } let order = 'relevance'; if (sort === 'date') order = 'date'; else if (sort === 'views') order = 'viewCount'; const perPage = Math.min(Math.max(1, Number(limit || 10)), 50); const targetPage = Math.max(1, Number(page || 1)); let pageToken = ''; let currentPage = 1; let lastItems = []; while (currentPage <= targetPage) { const params = { part: 'snippet', q, type: 'video', maxResults: String(perPage), order, videoEmbeddable: 'true', safeSearch: 'moderate', }; if (pageToken) params.pageToken = pageToken; const data = await ytFetchJson('https://www.googleapis.com/youtube/v3/search', params); if (currentPage === targetPage) { lastItems = Array.isArray(data.items) ? data.items : []; break; } const next = data.nextPageToken; if (!next) { lastItems = []; break; } pageToken = String(next); currentPage++; } const videoIds = (lastItems || []).map((item) => item?.id?.videoId).filter(Boolean); const detailsMap = new Map(); if (videoIds.length > 0) { try { const detailsData = await ytFetchJson('https://www.googleapis.com/youtube/v3/videos', { part: 'contentDetails,statistics,status', id: videoIds.join(','), }); for (const vid of detailsData?.items || []) if (vid?.id) detailsMap.set(vid.id, vid); } catch (e) { console.warn('[YouTube] details fetch failed, continuing without durations:', e?.message || e); } } return (lastItems || []).map((item) => { const videoId = item?.id?.videoId; const snippet = item?.snippet || {}; const thumb = snippet.thumbnails?.high?.url || snippet.thumbnails?.medium?.url || snippet.thumbnails?.default?.url || undefined; const details = videoId ? detailsMap.get(videoId) : null; const duration = parseISODurationToSeconds(details?.contentDetails?.duration || ''); const views = details?.statistics?.viewCount != null ? Number(details.statistics.viewCount) : undefined; const channelId = snippet.channelId || undefined; const embeddable = details?.status ? details.status.embeddable !== false : undefined; return { title: snippet.title || '', id: videoId, url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined, thumbnail: thumb, uploaderName: snippet.channelTitle || undefined, type: 'video', duration: duration > 0 ? duration : undefined, views, publishedAt: snippet.publishedAt || undefined, channelId, channelHandle: snippet.channelTitle || undefined, channelExternalId: channelId, channelUrl: channelId ? `https://www.youtube.com/channel/${channelId}` : undefined, embeddable, }; }); } /** @type {{ id: 'yt', label: string, search: (q: string, opts: { limit: number, page?: number, sort?: 'relevance'|'date'|'views' }) => Promise }} */ const handler = { id: 'yt', label: 'YouTube', async search(q, opts) { const { limit = 10, page = 1, sort = 'relevance' } = opts || {}; const mode = getSearchMode(); const ttl = getScrapeTtlMs(); const perPage = Math.min(Math.max(1, Number(limit || 10)), 50); const key = `yt|${hashSearchKey(`${String(q).toLowerCase().trim()}|${perPage}|${page}|${sort}|${mode}`)}`; // 1) mémoire const memHit = memGet(key, ttl); if (memHit) { ytMetrics.scrapeHits++; return memHit; } // 2) SQLite try { const { getCachedYoutubeSearch } = await import('../db.mjs'); const cached = getCachedYoutubeSearch(key); if (cached?.items) { ytMetrics.scrapeHits++; memSet(key, cached.items); return cached.items; } } catch {} const persist = async (items, source) => { // Ne jamais persister un résultat vide : une page vide transitoire // (continuation expirée, raté réseau partiel) ne doit pas empoisonner // le cache et bloquer les requêtes suivantes. if (!Array.isArray(items) || items.length === 0) return items; memSet(key, items); try { const { setCachedYoutubeSearch, incYoutubeMetrics } = await import('../db.mjs'); setCachedYoutubeSearch(key, q, items, source, ttl); if (source === 'scrape') incYoutubeMetrics({ scrapeCalls: 1 }); } catch {} return items; }; const t0 = Date.now(); const tryScrape = async () => { ytMetrics.scrapeCalls++; try { const { incYoutubeMetrics } = await import('../db.mjs'); incYoutubeMetrics({ scrapeCalls: 1 }); } catch {} return searchViaScrape(q, { limit: perPage, page, sort }); }; const tryApi = () => searchViaApi(q, { limit: perPage, page, sort }); const tryInnerTube = () => searchViaInnerTube(q, { limit: perPage, page, sort }); const log = (source, extra = '') => console.log(`[YT search] source=${source} mode=${mode} latency=${Date.now() - t0}ms results=${extra}`); try { if (mode === 'api-only') { const items = await tryApi(); log('api', items.length); return persist(items, 'api'); } if (mode === 'scrape-only') { const items = await tryScrape(); log('scrape', items.length); return persist(items, 'scrape'); } if (mode === 'innertube-only') { const items = await tryInnerTube(); log('innertube', items.length); return persist(items, 'innertube'); } if (mode === 'api-first') { try { const items = await tryApi(); log('api', items.length); return persist(items, 'api'); } catch (apiErr) { ytMetrics.fallbacks++; console.warn(`[YT search] api-first fallback to scrape: ${apiErr?.message || apiErr}`); const items = await tryScrape(); log('scrape(fallback)', items.length); return persist(items, 'scrape'); } } if (mode === 'scrape-first') { // scrape -> API (comportement historique, sans InnerTube) try { const items = await tryScrape(); log('scrape', items.length); return persist(items, 'scrape'); } catch (scrapeErr) { ytMetrics.scrapeErrors++; ytMetrics.fallbacks++; console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`); try { const items = await tryApi(); log('api(fallback)', items.length); return persist(items, 'api'); } catch (apiErr) { const noBinary = scrapeErr?.code === 'yt_scrape_no_binary'; const noKey = apiErr?.code === 'youtube_api_key_unavailable'; if (noBinary && noKey) { const err = new Error('YouTube indisponible : yt-dlp introuvable (définissez YT_DLP_PATH) et aucune clé YOUTUBE_API_KEY configurée.'); err.ytStatus = 503; err.code = 'youtube_no_source'; throw err; } console.error(`[YT search] both sources failed. scrape=${scrapeErr?.message} api=${apiErr?.message}`); throw scrapeErr; } } } // innertube-first (défaut) : InnerTube -> scrape yt-dlp -> API officielle. // Chaque couche ne fait jamais échouer la recherche à elle seule. const errors = []; try { const items = await tryInnerTube(); log('innertube', items.length); return persist(items, 'innertube'); } catch (itErr) { ytMetrics.innertubeErrors = (ytMetrics.innertubeErrors || 0) + 1; ytMetrics.fallbacks++; errors.push(`innertube=${itErr?.message}`); console.warn(`[YT search] innertube failed (${itErr?.code || 'unknown'}), fallback to scrape`); } try { const items = await tryScrape(); log('scrape(fallback)', items.length); return persist(items, 'scrape'); } catch (scrapeErr) { ytMetrics.scrapeErrors++; ytMetrics.fallbacks++; errors.push(`scrape=${scrapeErr?.message}`); console.warn(`[YT search] scrape failed (${scrapeErr?.code || 'unknown'}), fallback to api`); } try { const items = await tryApi(); log('api(fallback)', items.length); return persist(items, 'api'); } catch (apiErr) { errors.push(`api=${apiErr?.message}`); const noSource = !getYouTubeKeys().length; if (noSource) { const err = new Error(`YouTube indisponible (${errors.join(' | ').slice(0, 220)}). Vérifiez le réseau/YT_DLP_PATH ou ajoutez YOUTUBE_API_KEYS.`); err.ytStatus = 503; err.code = 'youtube_no_source'; throw err; } console.error(`[YT search] all sources failed. ${errors.join(' | ')}`); const err = new Error(errors[0] || 'search_failed'); err.ytStatus = 502; err.code = 'youtube_all_sources_failed'; throw err; } } catch (error) { const status = error?.ytStatus || 500; console.error(`YouTube search error (status ${status}):`, error?.message || error); throw error; } }, async suggest(q, opts) { const limit = Math.min(20, Math.max(1, Number(opts?.limit || 10))); const query = String(q || '').trim(); if (query.length < 2) return []; try { const params = new URLSearchParams({ client: 'youtube', ds: 'yt', hl: 'fr', q: query }); const headers = { 'User-Agent': 'Mozilla/5.0' }; const proxy = String(process.env.YT_EGRESS_PROXY || '').trim(); const fetchOpts = { headers, signal: AbortSignal.timeout(6000) }; // Note : suggest reste sans proxy (endpoint Google public léger) sauf si egress configuré côté infra. if (proxy) console.debug?.('[YT suggest] egress proxy configured, direct fetch kept for suggest'); const resp = await fetch(`https://suggestqueries.google.com/complete/search?${params.toString()}`, fetchOpts); if (!resp.ok) return []; const text = await resp.text(); return parseYoutubeSuggestResponse(text).slice(0, limit); } catch { return []; } } }; export default handler;