diff --git a/server/index.mjs b/server/index.mjs index 934ffd6..855085a 100644 --- a/server/index.mjs +++ b/server/index.mjs @@ -17,6 +17,7 @@ import axios from 'axios'; import rumbleRouter from './rumble.mjs'; import { providerRegistry, validateProviders } from './providers/registry.mjs'; import { dedupeSuggestGroups } from './suggest.mjs'; +import { fetchWebSuggest, fetchOdyseeLighthouseSuggest } from './suggest-web.mjs'; import { pickTrack, parseTrackText, parseVtt, dedupeTranscriptLines, orderedTracks, translatedFallbacks, firstPerLanguage, normalizeTranscriptProvider, transcriptTrackExt, looksLikeHtmlError } from './transcript.mjs'; import { getUserByUsername, @@ -2596,6 +2597,13 @@ app.get('/api/search/suggest', suggestLimiter, async (req, res) => { const cacheKey = `suggest:${validProviders.join(',')}:${q.toLowerCase()}:${limit}`; const cached = suggestCacheGet(cacheKey); if (cached) return res.json(cached); + // Enrichissement gratuit (sans clé) lancé en parallèle du fan-out + // providers : suggestions web génériques (Bing + Google) + vrais titres + // vidéo Odysee (Lighthouse) quand `od` est demandé. Jamais bloquant. + const webEnabled = String(process.env.SUGGEST_WEB_ENABLED ?? '1') !== '0'; + const wantOdysee = validProviders.includes('od'); + const webPromise = webEnabled ? fetchWebSuggest(q, { limit }) : Promise.resolve([]); + const lighthousePromise = wantOdysee ? fetchOdyseeLighthouseSuggest(q, { limit }) : Promise.resolve([]); const results = await Promise.allSettled( validProviders.map((providerId) => { const mod = providerRegistry[providerId]; @@ -2603,6 +2611,7 @@ app.get('/api/search/suggest', suggestLimiter, async (req, res) => { return Promise.resolve().then(() => mod.suggest(q, { limit })); }) ); + const [webRes, lightRes] = await Promise.allSettled([webPromise, lighthousePromise]); const groups = {}; results.forEach((result, index) => { const providerId = validProviders[index]; @@ -2615,8 +2624,14 @@ app.get('/api/search/suggest', suggestLimiter, async (req, res) => { groups[providerId] = []; } }); - // La même occurrence n'est renvoyée qu'une fois (premier provider gagne). - const data = { q, groups: dedupeSuggestGroups(groups, validProviders) }; + if (wantOdysee && lightRes.status === 'fulfilled' && Array.isArray(lightRes.value) && lightRes.value.length > 0) { + groups.od = [...(groups.od || []), ...lightRes.value].slice(0, limit); + } + if (webEnabled && webRes.status === 'fulfilled' && Array.isArray(webRes.value) && webRes.value.length > 0) { + groups.web = webRes.value.slice(0, limit); + } + // La même occurrence n'est renvoyée qu'une fois (providers d'abord, web en dernier). + const data = { q, groups: dedupeSuggestGroups(groups, [...validProviders, 'web']) }; suggestCacheSet(cacheKey, data); return res.json(data); } catch (e) { diff --git a/server/suggest-web.mjs b/server/suggest-web.mjs new file mode 100644 index 0000000..d4bbe4c --- /dev/null +++ b/server/suggest-web.mjs @@ -0,0 +1,130 @@ +// Web + Odysee suggestion enrichment (free APIs, no key required). +// Pure parsers are exported for offline unit tests; fetchers never throw. + +const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'; + +function webTimeoutMs() { + const n = Number(process.env.SUGGEST_WEB_TIMEOUT_MS || 4000); + return Number.isFinite(n) && n > 0 ? Math.min(n, 15000) : 4000; +} + +/** + * Parse an OSJSON response (`["q", ["s1", "s2", ...], ...]`) shared by + * Bing (`osjson.aspx`) and Google (`client=chrome`). Garbage yields []. + */ +export function parseOsjsonList(text) { + try { + const data = JSON.parse(String(text || '')); + const list = Array.isArray(data) ? data[1] : null; + if (!Array.isArray(list)) return []; + return list.map((s) => String(s ?? '').trim()).filter(Boolean); + } catch { + return []; + } +} + +/** Bing autocomplete (no key): fr suggestions, up to `limit`. */ +export async function fetchBingSuggest(q, { limit = 10, lang = 'fr' } = {}) { + const query = String(q || '').trim(); + if (query.length < 2) return []; + try { + const params = new URLSearchParams({ query, language: String(lang || 'fr') }); + const resp = await fetch(`https://api.bing.com/osjson.aspx?${params.toString()}`, { + headers: { 'User-Agent': UA, Accept: 'application/json' }, + signal: AbortSignal.timeout(webTimeoutMs()), + }); + if (!resp.ok) return []; + return parseOsjsonList(await resp.text()).slice(0, limit); + } catch { + return []; + } +} + +/** Google autocomplete, `client=chrome` flavour (no key). Same infra family + * as the YouTube suggest already used, but generic web suggestions. */ +export async function fetchGoogleChromeSuggest(q, { limit = 10, lang = 'fr' } = {}) { + const query = String(q || '').trim(); + if (query.length < 2) return []; + try { + const params = new URLSearchParams({ client: 'chrome', q: query, hl: String(lang || 'fr') }); + const resp = await fetch(`https://suggestqueries.google.com/complete/search?${params.toString()}`, { + headers: { 'User-Agent': UA }, + signal: AbortSignal.timeout(webTimeoutMs()), + }); + if (!resp.ok) return []; + return parseOsjsonList(await resp.text()).slice(0, limit); + } catch { + return []; + } +} + +/** Turn an Odysee/LBRY claim `name` slug into a readable suggestion. */ +export function humanizeClaimName(name) { + return String(name || '').replace(/[-_]+/g, ' ').replace(/\s+/g, ' ').trim(); +} + +/** + * Parse a Lighthouse (Odysee search API, no key) response into readable + * video-title suggestions. + */ +export function parseLighthouseList(data, limit = 10) { + const arr = Array.isArray(data) ? data : []; + const out = []; + for (const item of arr) { + const t = humanizeClaimName(item?.name); + if (t && t.length >= 3) out.push(t); + if (out.length >= limit) break; + } + return out; +} + +/** Odysee Lighthouse full-text search (no key): real Odysee video titles. */ +export async function fetchOdyseeLighthouseSuggest(q, { limit = 10 } = {}) { + const query = String(q || '').trim(); + if (query.length < 2) return []; + try { + const params = new URLSearchParams({ s: query, size: String(Math.min(20, Math.max(1, limit))), from: '0' }); + const resp = await fetch(`https://lighthouse.odysee.com/search?${params.toString()}`, { + headers: { 'User-Agent': UA, Accept: 'application/json' }, + signal: AbortSignal.timeout(webTimeoutMs()), + }); + if (!resp.ok) return []; + return parseLighthouseList(await resp.json(), limit); + } catch { + return []; + } +} + +function normKey(s) { + return String(s ?? '').trim().toLowerCase().replace(/\s+/g, ' '); +} + +/** + * Combined generic web suggestions (Bing + Google chrome, in parallel), + * deduplicated (Bing first). Never throws. + */ +export async function fetchWebSuggest(q, { limit = 10, lang = 'fr' } = {}) { + const query = String(q || '').trim(); + if (query.length < 2) return []; + const lim = Math.min(20, Math.max(1, Number(limit) || 10)); + try { + const [bing, chrome] = await Promise.all([ + fetchBingSuggest(query, { limit: lim, lang }), + fetchGoogleChromeSuggest(query, { limit: lim, lang }), + ]); + const seen = new Set(); + const out = []; + for (const t of [...(bing || []), ...(chrome || [])]) { + const clean = String(t ?? '').trim(); + if (!clean) continue; + const k = normKey(clean); + if (seen.has(k)) continue; + seen.add(k); + out.push(clean); + if (out.length >= lim) break; + } + return out; + } catch { + return []; + } +} diff --git a/server/tests/suggest.test.mjs b/server/tests/suggest.test.mjs index 23ee43a..c6ef8ee 100644 --- a/server/tests/suggest.test.mjs +++ b/server/tests/suggest.test.mjs @@ -21,6 +21,7 @@ const yt = (await import('../providers/youtube.mjs')).default; const { parseYoutubeSuggestResponse } = await import('../providers/youtube.mjs'); const dm = (await import('../providers/dailymotion.mjs')).default; const { dedupeSuggestGroups } = await import('../suggest.mjs'); +const { parseOsjsonList, parseLighthouseList, humanizeClaimName } = await import('../suggest-web.mjs'); // Bare JSON payload expect( @@ -68,6 +69,25 @@ logOk('suggest min-length guard (no network)'); logOk('dedupeSuggestGroups cross-provider (no duplicates)'); } +// Web enrichment parsers (offline): Bing/Google OSJSON + Odysee Lighthouse +{ + expect( + JSON.stringify(parseOsjsonList('["tut",["tutoriel angular","tutoriel android"],[],{}]')) === + JSON.stringify(['tutoriel angular', 'tutoriel android']), + 'OSJSON payload parses', + ); + expect(parseOsjsonList('garbage').length === 0, 'OSJSON garbage yields []'); + expect(parseOsjsonList('["tut","notalist"]').length === 0, 'OSJSON non-list yields []'); + logOk('parseOsjsonList (Bing/Google)'); + expect(humanizeClaimName('tutoriel-express_nettoyage--moulinet') === 'tutoriel express nettoyage moulinet', 'claim slug humanized'); + expect( + JSON.stringify(parseLighthouseList([{ name: 'tutoriel-express' }, { name: 'x' }, { name: '' }], 10)) === + JSON.stringify(['tutoriel express']), + 'lighthouse list keeps readable titles only', + ); + logOk('parseLighthouseList (Odysee)'); +} + // ---------- 2) API contract against a real isolated server ---------- const PORT = await new Promise((resolve) => { const srv = net.createServer(); @@ -114,12 +134,14 @@ try { expect(res.status === 400, `expected 400, got ${res.status}`); }); - await scenario('providers=yt returns only yt suggestions', async () => { + await scenario('providers=yt returns only yt suggestions (+optional web enrichment)', async () => { const res = await fetch(`${baseUrl}/api/search/suggest?q=tutoriel&providers=yt&limit=5`); expect(res.status === 200, `expected 200, got ${res.status}`); const body = await res.json(); expect(body.q === 'tutoriel', 'q echoed'); - expect(JSON.stringify(Object.keys(body.groups)) === JSON.stringify(['yt']), `only yt group (got ${Object.keys(body.groups)})`); + const keys = Object.keys(body.groups); + expect(keys.includes('yt'), `yt group present (got ${keys})`); + expect(keys.every((k) => k === 'yt' || k === 'web'), `only yt (+web) groups (got ${keys})`); expect(Array.isArray(body.groups.yt), 'yt group is an array'); expect(body.groups.yt.length <= 5, 'limit respected'); }); @@ -129,7 +151,24 @@ try { expect(res.status === 200, `expected 200, got ${res.status}`); const body = await res.json(); const keys = Object.keys(body.groups).sort(); - expect(JSON.stringify(keys) === JSON.stringify(['dm', 'od', 'pt', 'ru', 'tw', 'yt']), `full registry fallback (got ${keys})`); + const withoutWeb = keys.filter((k) => k !== 'web'); + expect(JSON.stringify(withoutWeb) === JSON.stringify(['dm', 'od', 'pt', 'ru', 'tw', 'yt']), `full registry fallback (got ${keys})`); + }); + + await scenario('no duplicate occurrence across groups (web enrichment included)', async () => { + const res = await fetch(`${baseUrl}/api/search/suggest?q=tutoriel&limit=10`); + expect(res.status === 200, `expected 200, got ${res.status}`); + const body = await res.json(); + const seen = new Set(); + for (const list of Object.values(body.groups)) { + for (const raw of Array.isArray(list) ? list : []) { + const k = String(raw || '').trim().toLowerCase().replace(/\s+/g, ' '); + expect(!seen.has(k), `duplicate occurrence across groups: "${raw}"`); + seen.add(k); + } + } + const total = Object.values(body.groups).reduce((n, l) => n + (Array.isArray(l) ? l.length : 0), 0); + expect(total > 0, 'enrichment yields at least one suggestion'); }); } finally { server.kill(); diff --git a/src/app/search/suggest.spec.ts b/src/app/search/suggest.spec.ts index cb5583d..91e4118 100644 --- a/src/app/search/suggest.spec.ts +++ b/src/app/search/suggest.spec.ts @@ -45,8 +45,7 @@ function logOk(msg: string): void { logOk('dedupeSort alpha fallback'); // --- Pure helpers: merge groups --- - const merged = mergeGroups({ dm: ['Tutoriel android'], yt: ['Tutoriel Angular', 'tutoriel android'], xx: ['zz'] }); - assertEqual( + const merged = mergeGroups({ dm: ['Tutoriel android'], yt: ['Tutoriel Angular', 'tutoriel android'], xx: ['zz'] }); assertEqual( merged, [ { text: 'Tutoriel Angular', provider: 'yt' }, @@ -57,6 +56,18 @@ function logOk(msg: string): void { ); logOk('mergeGroups order + dedup'); + // Web enrichment group renders last and stays deduped against providers + const mergedWeb = mergeGroups({ web: ['Tutoriel angular', 'tutoriel cuisine'], yt: ['Tutoriel Angular'] }); + assertEqual( + mergedWeb, + [ + { text: 'Tutoriel Angular', provider: 'yt' }, + { text: 'tutoriel cuisine', provider: 'web' }, + ], + 'mergeGroups keeps web enrichment last with no duplicates', + ); + logOk('mergeGroups web tier'); + // --- Pure helpers: dedupeStrings (whitespace + case variants) --- assertEqual( dedupeStrings([' Tutoriel Angular ', 'tutoriel angular', 'TUTORIEL ANGULAR', 'autre', '', ' ']), diff --git a/src/components/search/search-box.component.ts b/src/components/search/search-box.component.ts index 7579796..929beb4 100644 --- a/src/components/search/search-box.component.ts +++ b/src/components/search/search-box.component.ts @@ -318,6 +318,7 @@ export class SearchBoxComponent { providerBadge(pid: string | null): string { if (!pid) return ''; + if (pid === 'web') return 'Web'; const found = PROVIDERS.find((p) => p.id === (pid as ProviderId)); return found ? found.shortLabel : String(pid).toUpperCase(); }