feat(search): panneau de filtres unifié, clavier/focus du typeahead
CI / build-and-test (push) Successful in 14m10s

Barre de recherche
- Panneau de filtres unifié (remplace le ProviderPicker) : sources, type
  (vidéos/shorts/direct/chaînes), période (heure/jour/semaine/mois/année),
  durée (<4 / 4-20 / >20 min) et tri. Pastilles + roving tabindex
  (←/→ dans un groupe, ↑/↓ entre groupes), focus trap, Échap, Ctrl/⌘+Maj+F,
  recherches récentes, « mémoriser par défaut », « Réinitialiser ».
- Pastilles des filtres actifs sous la barre (retrait en un clic) + compteur.
- Typeahead : un seul keydown (fin du double Enter), ↑/↓ bouclants,
  Home/End, PageUp/PageDown, Entrée valide la ligne surlignée (sinon
  recherche brute), Tab complète sans chercher, Échap ferme puis vide,
  ligne « Rechercher <q> », défilement de l'option active, aria à jour.
- Focus : le panneau se referme dès que le focus quitte la barre
  (focusout + relatedTarget + activeElement), pointerdown neutralisé sur les
  lignes pour garder le focus dans l'input (plus de scintillement).
- Opérateurs en clair : `linux live:`, `tuto today: long:` deviennent des
  filtres et sont retirés de la requête (autocomplétion après le `:`).

Filtres côté serveur
- Modèle partagé : src/app/search/filters.ts + server/search-filters.mjs.
- /api/search?type=&duration=&period=&sort= : InnerTube (upload_date, type,
  duration, features), Data API v3 (type, videoDuration, publishedAfter),
  yt-dlp --dateafter ; post-filtrage pour les providers sans filtre natif.
  Un champ manquant ne fait jamais disparaître un résultat.
- Filtres dans l'URL (partageables) + adapters + SearchService (clé de cache).
- Correction : la recherche Shorts renvoyait 0 (ShortsLockupView/GridShelfView
  non mappés, id/titre dans l'endpoint) -> mappage + shelf + vignette/vues.

Tests / docs
- 23 tests unitaires (clavier, focus, opérateurs, panneau de filtres),
  server/tests/search-filters.test.mjs (npm run test:filters) + CI,
  scénarios e2e des filtres, README/api/README/todo/MCP à jour.
This commit is contained in:
2026-09-28 17:42:19 -04:00
parent 9fb0ce7549
commit f1f673040e
40 changed files with 3111 additions and 938 deletions
+17 -6
View File
@@ -17,6 +17,7 @@ import axios from 'axios';
import rumbleRouter from './rumble.mjs';
import { providerRegistry, validateProviders } from './providers/registry.mjs';
import { dedupeSuggestGroups } from './suggest.mjs';
import { parseSearchFilters, activeFilters as activeSearchFilters, applySearchFilters, filtersCacheKey } from './search-filters.mjs';
import { fetchWebSuggest, fetchOdyseeLighthouseSuggest } from './suggest-web.mjs';
import { pickTrack, parseTrackText, parseVtt, dedupeTranscriptLines, orderedTracks, translatedFallbacks, firstPerLanguage, normalizeTranscriptProvider, transcriptTrackExt, looksLikeHtmlError, ensureFmtParam, isEmptyTimedTextBody, isBlobTranscript, mergeTranscriptCandidates } from './transcript.mjs';
import {
@@ -2522,7 +2523,7 @@ r.post('/telemetry/events', authMiddleware, telemetryLimiter, (req, res) => {
const { event, meta } = req.body || {};
if (!event || typeof event !== 'string') return res.status(400).json({ error: 'event_required' });
// Whitelist known event names to keep the table clean
const allowed = new Set(['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open', 'suggest_shown', 'suggest_used']);
const allowed = new Set(['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open', 'filter_panel_open', 'filter_apply', 'suggest_shown', 'suggest_used']);
if (!allowed.has(event)) return res.status(400).json({ error: 'unknown_event' });
const row = insertTelemetryEvent({ userId: req.user.id, event, meta: (meta && typeof meta === 'object') ? meta : null });
return res.status(201).json(row || { ok: true });
@@ -2542,7 +2543,7 @@ r.get('/telemetry/events', authMiddleware, (req, res) => {
r.get('/telemetry/summary', authMiddleware, (req, res) => {
const since = typeof req.query.since === 'string' ? req.query.since : undefined;
const events = {};
for (const name of ['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open', 'suggest_shown', 'suggest_used']) {
for (const name of ['search_submit', 'provider_picker_open', 'provider_apply', 'at_autocomplete_use', 'quick_menu_open', 'filter_panel_open', 'filter_apply', 'suggest_shown', 'suggest_used']) {
events[name] = countTelemetryEvents({ event: name, since });
}
return res.json({ events });
@@ -3105,10 +3106,15 @@ app.get('/api/search', async (req, res) => {
}
const pageNum = Math.max(1, Number(req.query.page || 1));
const pageSize = Math.min(50, Math.max(1, Number(req.query.pageSize || 24)));
// Filtres de recherche (type / durée / période / tri). Un filtre inconnu est
// ignoré (jamais 400) : `normalizeFilters` retombe sur 'all'.
const filters = parseSearchFilters(req.query);
// Optional sort parameter (normalized to known set)
const allowedSort = new Set(['relevance', 'date', 'views']);
let sort = (typeof req.query.sort === 'string' ? req.query.sort.trim().toLowerCase() : 'relevance');
if (!allowedSort.has(sort)) sort = 'relevance';
// `filters.sort` fait autorité (le front envoie les deux, ils concordent)
sort = filters.sort;
// Validate and normalize providers list (default to all supported when none/invalid)
const requested = typeof providers === 'string' ? String(providers) : '';
const validProviders = validateProviders(requested);
@@ -3118,8 +3124,8 @@ app.get('/api/search', async (req, res) => {
validProviders.map((providerId) => {
const mod = providerRegistry[/** @type {any} */(providerId)];
if (!mod || typeof mod.search !== 'function') return Promise.resolve([]);
// Basic options include pagination and sort hints
return Promise.resolve().then(() => mod.search(q, { limit: pageSize, page: pageNum, sort }));
// Basic options include pagination, sort hints and the search filters
return Promise.resolve().then(() => mod.search(q, { limit: pageSize, page: pageNum, sort, filters }));
})
);
@@ -3129,7 +3135,8 @@ app.get('/api/search', async (req, res) => {
results.forEach((result, index) => {
const providerId = validProviders[index];
if (result.status === 'fulfilled') {
groups[providerId] = Array.isArray(result.value) ? result.value : [];
// Filtres non supportés nativement par le provider -> affinage ici.
groups[providerId] = applySearchFilters(providerId, Array.isArray(result.value) ? result.value : [], filters);
} else {
console.warn(`Search failed for provider ${providerId}:`, result.reason?.message || result.reason);
groups[providerId] = [];
@@ -3144,7 +3151,11 @@ app.get('/api/search', async (req, res) => {
}
});
return res.json({ q, providers: validProviders, groups, errors, page: pageNum, pageSize, sort });
return res.json({
q, providers: validProviders, groups, errors,
page: pageNum, pageSize, sort,
filters: activeSearchFilters(filters),
});
} catch (e) {
return res.status(500).json({ error: 'search_failed', details: String(e?.message || e) });
}
+104 -8
View File
@@ -2,6 +2,7 @@
// WEB client : search + continuations (pagination illimitée), watch-next (related),
// sans clé API ni quota. Le dispatcher bascule sur scrape/API si indisponible.
import { hashSearchKey, ytMetrics } from './youtube-common.mjs';
import { innertubeSearchFilters, filtersCacheKey } from '../search-filters.mjs';
let sessionPromise = null;
let LogSilenced = false;
@@ -95,22 +96,53 @@ export function mapVideoNode(n) {
if (type === 'LockupView') return mapLockupView(n);
if (/playlist|channel|gridchannel|shelf|radio|show|album/i.test(type)
&& !/playlistpanelvideo|watchcard/i.test(type)) return null;
const id = n.id ? String(n.id) : null;
// Les Shorts (ShortsLockupView / ReelItem) n'ont pas de `id` : il vit dans
// l'endpoint de navigation. Sans cela, une recherche `type=shorts` ne
// remonterait aucun résultat.
const entityId = n.entity_id ? String(n.entity_id) : '';
const id = n.id ? String(n.id) : (
n.on_tap_endpoint?.payload?.videoId
|| n.inline_player_data?.payload?.videoId
|| n.on_tap?.innertubeCommand?.reelWatchEndpoint?.videoId
|| n.on_tap?.innertubeCommand?.reelPlayerEndpoint?.videoId
|| n.navigation_endpoint?.watchEndpoint?.videoId
|| n.innertubeCommand?.reelWatchEndpoint?.videoId
// `shorts-shelf-item-<videoId>` : dernier recours.
|| (entityId.startsWith('shorts-shelf-item-') ? entityId.slice('shorts-shelf-item-'.length) : '')
|| null
);
if (!id) return null;
const title = n.title?.text ?? (typeof n.title === 'string' ? n.title : '');
// `??` ne suffit pas : `typeof n.title === 'string' ? … : ''` renvoie ''
// (non nullish) et court-circuiterait les replis. On chaîne donc à part.
const title = n.title?.text
|| (typeof n.title === 'string' ? n.title : '')
// ShortsLockupView / ReelItem : le titre est dans les métadonnées superposées.
|| n.overlay_metadata?.primary_text?.text
|| n.overlay_metadata?.headline?.text
|| '';
if (!title) return null;
const duration = n.duration?.seconds != null ? Number(n.duration.seconds) : undefined;
const author = n.author || n.uploader || null;
const authorName = author?.name ?? (typeof author === 'string' ? author : undefined);
const authorId = author?.id ? String(author.id) : undefined;
const views = parseViewsText(n.view_count?.text ?? n.view_count ?? n.views?.text);
const views = parseViewsText(
n.view_count?.text ?? n.view_count ?? n.views?.text
// ShortsLockupView : vues dans la 2e ligne des métadonnées superposées.
?? n.overlay_metadata?.secondary_text?.text,
);
// Vignette : `thumbnails[]` (classique), `thumbnail` (ShortsLockupView :
// tableau de Thumbnail) ou la vignette portée par l'endpoint du Short.
const thumb = bestThumb(n.thumbnails)
|| bestThumb(n.thumbnail)
|| bestThumb(n.on_tap_endpoint?.payload?.thumbnail?.thumbnails)
|| undefined;
const isShort = /short|reel/i.test(type) || (Number.isFinite(duration) && duration > 0 && duration <= 70 && /short/i.test(title) === false && /reel|short/i.test(type));
const badges = Array.isArray(n.badges) ? n.badges.map((b) => b?.label).filter(Boolean) : [];
return {
title: String(title),
id,
url: `https://www.youtube.com/watch?v=${id}`,
thumbnail: bestThumb(n.thumbnails),
thumbnail: thumb,
uploaderName: authorName,
type: 'video',
...(Number.isFinite(duration) && duration > 0 ? { duration } : {}),
@@ -190,6 +222,55 @@ export function mapNodes(nodes) {
return out;
}
/**
* Aplatit un nœud InnerTube (YTNode) en ses feuilles.
* `Shelf.content` peut être une `Grid`/`Shelf` imbriquée : les Shorts d'une
* recherche « Shorts » vivent là, hors de `feed.videos`.
*/
function flattenNodes(node, depth = 0, out = []) {
if (!node || depth > 5) return out;
if (Array.isArray(node)) {
for (const n of node) flattenNodes(n, depth, out);
return out;
}
if (typeof node !== 'object') return out;
let descended = false;
for (const key of ['contents', 'items', 'content']) {
const child = node[key];
if (child && typeof child === 'object') {
descended = true;
flattenNodes(child, depth + 1, out);
}
}
if (!descended) out.push(node);
return out;
}
/**
* Vidéos contenues dans les `Shelf` d'un feed de recherche (Shorts/clips).
* `feed.videos` ne regarde que les rendus directs, donc une recherche
* `type=shorts` y renvoie 0 sans ce complément.
* Pur et testable offline.
*/
export function mapShelfNodes(feed) {
const seen = new Set();
const out = [];
try {
const results = Array.isArray(feed?.results) ? Array.from(feed.results) : [];
for (const node of results) {
const type = String(node?.type || '');
if (!/shelf/i.test(type)) continue;
for (const leaf of flattenNodes(node.content ?? node)) {
const mapped = mapVideoNode(leaf);
if (!mapped || seen.has(mapped.id)) continue;
seen.add(mapped.id);
out.push(mapped);
}
}
} catch { return out; }
return out;
}
// Chaîne de continuations en mémoire : q_hash -> { feed, pages }.
// (SQLite persiste les pages déjà servies ; la chaîne évite de rejouer les pages 1..N-1.)
const chainCache = new Map();
@@ -212,6 +293,8 @@ function sortParam(sort) {
/**
* Recherche InnerTube avec vraie pagination (continuations).
* Page 1 ~35-40 vidéos ; pages suivantes via getContinuation() fusionnées (façon SmartTube).
* `opts.filters` : { type, duration, period, sort } — traduits en filtres
* natifs InnerTube (upload_date / type / duration / features).
*/
export async function searchViaInnerTube(q, opts = {}) {
const query = String(q || '').trim();
@@ -219,9 +302,16 @@ export async function searchViaInnerTube(q, opts = {}) {
const limit = Math.min(50, Math.max(1, Number(opts?.limit || 24)));
const page = Math.min(10, Math.max(1, Number(opts?.page || 1)));
const sort = String(opts?.sort || 'relevance');
const itFilters = innertubeSearchFilters(opts?.filters);
// Les Shorts d'une recherche « Shorts » sont servis dans un `Shelf`, hors de
// `feed.videos` : on complète le lot dans ce seul cas (une recherche normale
// garde exactement son comportement et son ordre actuels).
const withShelves = itFilters.type === 'shorts';
ytMetrics.innertubeCalls = (ytMetrics.innertubeCalls || 0) + 1;
const yt = await getSession();
const key = `it|${hashSearchKey(`${query.toLowerCase()}|${sort}`)}`;
// La chaîne de continuations est propre à une combinaison
// (requête + tri + filtres) : sinon on servirait les pages d'un filtre à l'autre.
const key = `it|${hashSearchKey(`${query.toLowerCase()}|${sort}|${filtersCacheKey(opts?.filters)}`)}`;
let entry = chainGet(key);
let feed = entry?.feed || null;
let collected = entry?.items ? [...entry.items] : [];
@@ -231,17 +321,23 @@ export async function searchViaInnerTube(q, opts = {}) {
// [start, start+limit[ (façon SmartTube qui remplit son écran au fil des
// continuations), au lieu d'aligner 1 page API = 1 page UI.
const need = page * limit;
const harvest = (f) => {
const direct = mapNodes(f.videos);
if (!withShelves) return direct;
const extra = mapShelfNodes(f).filter((m) => !direct.some((d) => d.id === m.id));
return direct.concat(extra);
};
try {
if (!feed) {
feed = await yt.search(query, { sort_by: sortParam(sort) });
collected = mapNodes(feed.videos);
feed = await yt.search(query, { ...itFilters, sort_by: sortParam(sort) });
collected = harvest(feed);
pagesDone = 1;
}
let guard = 0;
while (collected.length < need && feed?.has_continuation && guard < 12) {
guard++;
feed = await feed.getContinuation();
const fresh = mapNodes(feed.videos).filter((m) => !collected.some((c) => c.id === m.id));
const fresh = harvest(feed).filter((m) => !collected.some((c) => c.id === m.id));
collected = collected.concat(fresh);
pagesDone++;
if (!fresh.length) break;
+14 -3
View File
@@ -3,6 +3,7 @@
import { execFile as execFileCb } from 'node:child_process';
import { promisify } from 'node:util';
import { getYtDlpTimeoutMs, buildYtDlpExtraArgs, resolveYtDlpBin } from './youtube-common.mjs';
import { periodMs } from '../search-filters.mjs';
const execFileAsync = promisify(execFileCb);
@@ -85,7 +86,7 @@ export function classifyScrapeError(e) {
return Object.assign(new Error(`YouTube scrape failed: ${msg.slice(0, 200)}`), { ytStatus: 502, code: 'yt_scrape_failed' });
}
async function runYtDlpFlat(queryOrUrl, { limit = 10, playlistStart = 1 } = {}) {
async function runYtDlpFlat(queryOrUrl, { limit = 10, playlistStart = 1, extraArgs = [] } = {}) {
const bin = await resolveYtDlpBin();
const timeout = getYtDlpTimeoutMs();
const end = playlistStart + Math.max(1, Math.min(50, Number(limit || 10))) - 1;
@@ -98,6 +99,7 @@ async function runYtDlpFlat(queryOrUrl, { limit = 10, playlistStart = 1 } = {})
'--no-playlist',
'--playlist-start', String(playlistStart),
'--playlist-end', String(end),
...extraArgs,
...buildYtDlpExtraArgs(),
queryOrUrl,
];
@@ -112,7 +114,11 @@ async function runYtDlpFlat(queryOrUrl, { limit = 10, playlistStart = 1 } = {})
}
}
/** Recherche sans clé. sort: relevance|date|views */
/**
* Recherche sans clé. sort: relevance|date|views
* `opts.filters` : seule la période est traduite nativement (`--dateafter`) ;
* durée / type sont affinés en post-traitement (côté /api/search).
*/
export async function searchViaScrape(q, opts = {}) {
const query = String(q || '').trim();
if (query.length < 2) return [];
@@ -125,7 +131,12 @@ export async function searchViaScrape(q, opts = {}) {
if (sort === 'date') prefix = 'ytsearchdate';
// views : pas de préfixe natif stable -> ytsearch + tri local
const target = `${prefix}${perPage}:${query}`;
let items = await runYtDlpFlat(target, { limit: perPage, playlistStart: start });
// --dateafter : borne basse de mise en ligne (format YYYYMMDD attendu par yt-dlp).
const maxAge = periodMs(opts?.filters?.period);
const extraArgs = maxAge > 0
? ['--dateafter', new Date(Date.now() - maxAge).toISOString().slice(0, 10).replace(/-/g, '')]
: [];
let items = await runYtDlpFlat(target, { limit: perPage, playlistStart: start, extraArgs });
if (sort === 'views') items = [...items].sort((a, b) => (b.views || 0) - (a.views || 0));
return items.slice(0, perPage);
}
+33 -13
View File
@@ -15,6 +15,7 @@ import {
} from './youtube-common.mjs';
import { searchViaScrape } from './youtube-scrape.mjs';
import { searchViaInnerTube } from './youtube-innertube.mjs';
import { apiSearchParams, filtersCacheKey } from '../search-filters.mjs';
function parseISODurationToSeconds(iso) {
if (typeof iso !== 'string' || !iso) return 0;
@@ -108,7 +109,7 @@ export function ytScrapeCacheStats() {
return { memEntries: memCache.size, memMax: MEM_MAX };
}
async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance' } = {}) {
async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance', filters = null } = {}) {
const keys = getYouTubeKeys();
if (!keys.length) {
throw Object.assign(new Error('YOUTUBE_API_KEY not configured'), { ytStatus: 503, code: 'youtube_api_key_unavailable' });
@@ -116,6 +117,8 @@ async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance' } = {}
let order = 'relevance';
if (sort === 'date') order = 'date';
else if (sort === 'views') order = 'viewCount';
// Filtres natifs de l'API v3 : type, videoDuration, publishedAfter.
const extra = apiSearchParams(filters);
const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
const targetPage = Math.max(1, Number(page || 1));
let pageToken = '';
@@ -123,8 +126,9 @@ async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance' } = {}
let lastItems = [];
while (currentPage <= targetPage) {
const params = {
part: 'snippet', q, type: 'video', maxResults: String(perPage), order,
part: 'snippet', q, maxResults: String(perPage), order,
videoEmbeddable: 'true', safeSearch: 'moderate',
...extra,
};
if (pageToken) params.pageToken = pageToken;
const data = await ytFetchJson('https://www.googleapis.com/youtube/v3/search', params);
@@ -148,35 +152,51 @@ async function searchViaApi(q, { limit = 10, page = 1, sort = 'relevance' } = {}
}
return (lastItems || []).map((item) => {
const videoId = item?.id?.videoId;
// Une recherche de type `channel` renvoie des channelId, pas des videoId.
const rawId = videoId || item?.id?.channelId || null;
const isChannel = !videoId && !!item?.id?.channelId;
if (!rawId) return null;
const snippet = item?.snippet || {};
const thumb = snippet.thumbnails?.high?.url || snippet.thumbnails?.medium?.url || snippet.thumbnails?.default?.url || undefined;
const details = videoId ? detailsMap.get(videoId) : null;
const duration = parseISODurationToSeconds(details?.contentDetails?.duration || '');
const views = details?.statistics?.viewCount != null ? Number(details.statistics.viewCount) : undefined;
const channelId = snippet.channelId || undefined;
const channelId = snippet.channelId || item?.id?.channelId || undefined;
const embeddable = details?.status ? details.status.embeddable !== false : undefined;
return {
title: snippet.title || '', id: videoId,
url: videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined,
thumbnail: thumb, uploaderName: snippet.channelTitle || undefined, type: 'video',
title: snippet.title || '', id: rawId,
url: isChannel
? `https://www.youtube.com/channel/${rawId}`
: `https://www.youtube.com/watch?v=${rawId}`,
thumbnail: thumb, uploaderName: snippet.channelTitle || undefined,
type: isChannel ? 'channel' : 'video',
duration: duration > 0 ? duration : undefined, views,
publishedAt: snippet.publishedAt || undefined, channelId,
channelHandle: snippet.channelTitle || undefined, channelExternalId: channelId,
channelUrl: channelId ? `https://www.youtube.com/channel/${channelId}` : undefined, embeddable,
};
});
}).filter(Boolean);
}
/** @type {{ id: 'yt', label: string, search: (q: string, opts: { limit: number, page?: number, sort?: 'relevance'|'date'|'views' }) => Promise<Suggestion[]> }} */
/**
* @typedef {Object} SearchFilters
* @property {string} [type] all|video|shorts|live|channel
* @property {string} [duration] all|short|medium|long
* @property {string} [period] all|hour|today|week|month|year
* @property {string} [sort] relevance|date|views
*/
/** @type {{ id: 'yt', label: string, search: (q: string, opts: { limit: number, page?: number, sort?: 'relevance'|'date'|'views', filters?: SearchFilters }) => Promise<Suggestion[]> }} */
const handler = {
id: 'yt',
label: 'YouTube',
async search(q, opts) {
const { limit = 10, page = 1, sort = 'relevance' } = opts || {};
const { limit = 10, page = 1, sort = 'relevance', filters = null } = opts || {};
const mode = getSearchMode();
const ttl = getScrapeTtlMs();
const perPage = Math.min(Math.max(1, Number(limit || 10)), 50);
const key = `yt|${hashSearchKey(`${String(q).toLowerCase().trim()}|${perPage}|${page}|${sort}|${mode}`)}`;
// La signature des filtres entre dans la clé de cache : deux recherches
// identiques avec des filtres différents ne doivent pas se partager de cache.
const key = `yt|${hashSearchKey(`${String(q).toLowerCase().trim()}|${perPage}|${page}|${sort}|${mode}|${filtersCacheKey(filters)}`)}`;
// 1) mémoire
const memHit = memGet(key, ttl);
if (memHit) { ytMetrics.scrapeHits++; return memHit; }
@@ -207,10 +227,10 @@ const handler = {
const tryScrape = async () => {
ytMetrics.scrapeCalls++;
try { const { incYoutubeMetrics } = await import('../db.mjs'); incYoutubeMetrics({ scrapeCalls: 1 }); } catch {}
return searchViaScrape(q, { limit: perPage, page, sort });
return searchViaScrape(q, { limit: perPage, page, sort, filters });
};
const tryApi = () => searchViaApi(q, { limit: perPage, page, sort });
const tryInnerTube = () => searchViaInnerTube(q, { limit: perPage, page, sort });
const tryApi = () => searchViaApi(q, { limit: perPage, page, sort, filters });
const tryInnerTube = () => searchViaInnerTube(q, { limit: perPage, page, sort, filters });
const log = (source, extra = '') => console.log(`[YT search] source=${source} mode=${mode} latency=${Date.now() - t0}ms results=${extra}`);
try {
if (mode === 'api-only') {
+262
View File
@@ -0,0 +1,262 @@
/**
* Filtres de recherche — modèle partagé avec le front (`src/app/search/filters.ts`).
*
* Rôle : valider/normaliser les filtres reçus par `GET /api/search`, les
* traduire dans les paramètres natifs de chaque provider (InnerTube, Data API
* v3…) et, pour les providers qui ne savent pas filtrer, affiner la liste
* renvoyée.
*
* Principe de sûreté : un filtre inconnu est IGNORÉ (jamais d'erreur 4xx), et
* un champ métadonnée absent ne fait jamais disparaître un résultat.
*/
const TYPES = ['all', 'video', 'shorts', 'live', 'channel'];
const DURATIONS = ['all', 'short', 'medium', 'long'];
const PERIODS = ['all', 'hour', 'today', 'week', 'month', 'year'];
const SORTS = ['relevance', 'date', 'views'];
/** @returns {any} filtres normalisés (toujours les 4 clés) */
export function normalizeFilters(raw) {
const src = raw && typeof raw === 'object' ? raw : {};
const pick = (v, allowed, fallback) => {
const s = String(v ?? '').trim().toLowerCase();
return allowed.includes(s) ? s : fallback;
};
return {
type: pick(src.type, TYPES, 'all'),
duration: pick(src.duration, DURATIONS, 'all'),
period: pick(src.period, PERIODS, 'all'),
sort: pick(src.sort, SORTS, 'relevance'),
};
}
/** Extrait + normalise les filtres depuis une query string Express. */
export function parseSearchFilters(query) {
const q = query && typeof query === 'object' ? query : {};
return normalizeFilters({
type: q.type,
duration: q.duration,
period: q.period,
sort: q.sort,
});
}
/** Filtres actifs seulement (pour le cache + la réponse JSON). */
export function activeFilters(filters) {
const n = normalizeFilters(filters);
const out = {};
if (n.type !== 'all') out.type = n.type;
if (n.duration !== 'all') out.duration = n.duration;
if (n.period !== 'all') out.period = n.period;
if (n.sort !== 'relevance') out.sort = n.sort;
return out;
}
/** Signature courte et stable (cache mémoire / SQLite). */
export function filtersCacheKey(filters) {
const n = normalizeFilters(filters);
return `${n.type}.${n.duration}.${n.period}.${n.sort}`;
}
const HOUR = 3600 * 1000;
const DAY = 24 * HOUR;
/** Bornes de la période de mise en ligne, en ms. 0 = pas de borne. */
export function periodMs(period) {
switch (normalizeFilters({ period }).period) {
case 'hour': return HOUR;
case 'today': return DAY;
case 'week': return 7 * DAY;
case 'month': return 30 * DAY;
case 'year': return 365 * DAY;
default: return 0;
}
}
/** Bornes de durée en secondes. */
export function durationBounds(duration) {
switch (normalizeFilters({ duration }).duration) {
case 'short': return { min: 1, max: 240 };
case 'medium': return { min: 240, max: 1200 };
case 'long': return { min: 1200, max: Infinity };
default: return null;
}
}
// ---------------------------------------------------------------------------
// Post-filtrage (providers sans filtre natif)
// ---------------------------------------------------------------------------
/**
* Dimensions que l'on peut affiner en post-traitement, par provider.
* - Twitch : les chaînes n'ont pas de durée exploitable, et `live` est le seul
* type fiable (les VOD portent `type: 'video'`).
* - Odysee : pas de durée fiable sur la recherche.
* - Le type `channel` n'a de sens que pour YouTube (seul provider qui expose
* des chaines comme résultats de recherche) : ailleurs on ne filtre pas, sinon
* la recherche vide le résultat au lieu de le restreindre.
*/
const POST_FILTER_DIMS = {
yt: ['period', 'duration', 'type'],
dm: ['period', 'duration', 'type'],
tw: ['period', 'type'],
pt: ['period', 'duration', 'type'],
od: ['period', 'type'],
ru: ['period', 'duration', 'type'],
};
function num(item, keys) {
for (const k of keys) {
const v = item?.[k];
if (typeof v === 'number' && Number.isFinite(v)) return v;
if (typeof v === 'string' && v.trim() && Number.isFinite(Number(v))) return Number(v);
}
return 0;
}
export function itemDurationSec(item) {
const d = num(item, ['duration', 'durationSec', 'length', 'durationSeconds']);
if (d > 0) return d;
// "1:23:45" / "12:34"
const raw = item?.duration ?? item?.length ?? '';
const s = String(raw || '');
if (s.includes(':')) {
const parts = s.split(':').map((p) => Number(p));
if (parts.length && parts.every((p) => Number.isFinite(p))) {
return parts.reduce((acc, p) => acc * 60 + p, 0);
}
}
return 0;
}
export function itemPublishedTs(item) {
const raw = item?.publishedAt ?? item?.uploadedDate ?? item?.uploadDate
?? item?.published_at ?? item?.uploaded ?? item?.timestamp ?? item?.created_time;
if (typeof raw === 'number' && Number.isFinite(raw)) {
// Les scrapes yt-dlp donnent un timestamp en secondes.
return raw > 1e12 ? raw : raw * 1000;
}
if (typeof raw === 'string' && raw.trim()) {
const parsed = Date.parse(raw);
if (!Number.isNaN(parsed)) return parsed;
const asNum = Number(raw);
if (Number.isFinite(asNum) && asNum > 0) return asNum > 1e12 ? asNum : asNum * 1000;
}
return 0;
}
export function isLiveItem(item) {
if (!item) return false;
if (item.isLive === true) return true;
return String(item.type || '').toLowerCase() === 'live';
}
export function isShortItem(item) {
if (!item) return false;
if (item.isShort === true) return true;
if (String(item.kind || '').toLowerCase() === 'clip') return true;
const d = itemDurationSec(item);
return d > 0 && d <= 70 && !/short/i.test(String(item.title || ''));
}
export function isChannelItem(item) {
return String(item?.type || '').toLowerCase() === 'channel' && item?.isLive !== true;
}
/** Le provider expose-t-il un signal de type exploitable ? */
function hasTypeSignal(item) {
if (!item) return false;
if (item.isLive === true || item.isLive === false) return true;
if (item.isShort === true || item.isShort === false) return true;
if (String(item.type || '').trim()) return true;
if (String(item.kind || '').trim()) return true;
return false;
}
/** Un item respecte-t-il les filtres ? (champs absents => jamais écarté) */
export function matchesFilters(item, filters) {
const n = normalizeFilters(filters);
if (n.type === 'all' && n.duration === 'all' && n.period === 'all') return true;
const bounds = durationBounds(n.duration);
if (bounds) {
const d = itemDurationSec(item);
if (d > 0 && (d < bounds.min || d >= bounds.max)) return false;
}
const maxAge = periodMs(n.period);
if (maxAge > 0) {
const ts = itemPublishedTs(item);
if (ts > 0 && (Date.now() - ts) > maxAge) return false;
}
if (n.type === 'all') return true;
if (!hasTypeSignal(item)) return true; // provider muet : on ne peut pas exclure
if (n.type === 'live') return isLiveItem(item);
if (n.type === 'channel') return isChannelItem(item);
if (n.type === 'shorts') return isShortItem(item);
if (n.type === 'video') {
return !isLiveItem(item) && !isChannelItem(item) && !isShortItem(item);
}
return true;
}
/**
* Affine une liste de résultats pour un provider donné. Renvoie la liste
* d'origine quand le provider ne sait pas post-filtrer cette dimension.
* @param {string} providerId
* @param {any[]} items
* @param {any} filters
*/
export function applySearchFilters(providerId, items, filters) {
const n = normalizeFilters(filters);
if (!Array.isArray(items) || items.length === 0) return items || [];
if (n.type === 'all' && n.duration === 'all' && n.period === 'all') return items;
const dims = POST_FILTER_DIMS[providerId] || [];
const wanted = ['period', 'duration', 'type'].filter((d) => dims.includes(d) && n[d] && n[d] !== 'all');
if (wanted.length === 0) return items;
// `channel` : hors YouTube, on ne filtre pas (sinon résultat vide).
if (wanted.includes('type') && n.type === 'channel' && providerId !== 'yt') return items;
return items.filter((it) => matchesFilters(it, n));
}
// ---------------------------------------------------------------------------
// Mapping provider
// ---------------------------------------------------------------------------
/**
* Filtres youtubei.js (`yt.search`) : les enums protos sont des noms explicites
* (UNDER_THREE_MINS…), pas des alias 'short'/'medium'/'long'.
* @param {any} filters
*/
export function innertubeSearchFilters(filters) {
const n = normalizeFilters(filters);
const out = {};
if (n.period !== 'all' && ['today', 'week', 'month', 'year'].includes(n.period)) {
out.upload_date = n.period;
}
if (n.type === 'video' || n.type === 'channel') out.type = n.type;
if (n.type === 'shorts') out.type = 'shorts';
// `live` n'est pas un SearchType : YouTube l'expose comme feature.
if (n.type === 'live') out.features = ['live'];
if (n.duration !== 'all') {
out.duration = n.duration === 'short'
? 'UNDER_THREE_MINS'
: n.duration === 'medium' ? 'THREE_TO_TWENTY_MINS' : 'OVER_TWENTY_MINS';
}
return out;
}
/**
* Paramètres additionnels pour la Data API v3 (`youtube/v3/search`).
* `type` : 'shorts'/'live' n'existent pas côté API -> laissés au post-filtrage.
*/
export function apiSearchParams(filters) {
const n = normalizeFilters(filters);
const out = {};
if (n.type === 'video' || n.type === 'channel') out.type = n.type;
if (n.duration !== 'all') out.videoDuration = n.duration;
const maxAge = periodMs(n.period);
if (maxAge > 0) out.publishedAfter = new Date(Date.now() - maxAge).toISOString();
return out;
}
+189
View File
@@ -0,0 +1,189 @@
// Filtres de recherche — tests unitaires (offline, aucun réseau).
// Run with: npm run test:filters
// Couverture : normalisation, bornes, post-filtrage par provider, mapping
// InnerTube / Data API v3.
import {
normalizeFilters, parseSearchFilters, activeFilters, filtersCacheKey,
periodMs, durationBounds, matchesFilters, applySearchFilters,
innertubeSearchFilters, apiSearchParams, itemDurationSec, itemPublishedTs,
} from '../search-filters.mjs';
function expect(cond, msg) {
if (!cond) throw new Error(`Assertion failed: ${msg}`);
}
function eq(actual, expected, msg) {
const a = JSON.stringify(actual);
const b = JSON.stringify(expected);
if (a !== b) throw new Error(`Assertion failed: ${msg} (expected ${b}, got ${a})`);
}
function logOk(msg) { console.log(`✓ ${msg}`); }
// ---------- Normalisation ----------
eq(normalizeFilters(null), { type: 'all', duration: 'all', period: 'all', sort: 'relevance' }, 'null filters -> defaults');
eq(
normalizeFilters({ type: 'LIVE', duration: 'moyen', period: 'nope', sort: 'views' }),
{ type: 'live', duration: 'all', period: 'all', sort: 'views' },
'values are case-insensitive, unknown values fall back per key (no cross-contamination)',
);
eq(
parseSearchFilters({ type: 'shorts', duration: 'short', period: 'week', sort: 'date', junk: 'x' }),
{ type: 'shorts', duration: 'short', period: 'week', sort: 'date' },
'parseSearchFilters reads the Express query',
);
eq(
parseSearchFilters({ type: 'plaid' }).type,
'all',
'unknown type is ignored (never a 400)',
);
eq(activeFilters({ type: 'live', period: 'all' }), { type: 'live' }, 'activeFilters keeps only non-defaults');
eq(
filtersCacheKey({ type: 'live', sort: 'date' }),
'live.all.all.date',
'cache key signature is stable and ordered',
);
logOk('normalizeFilters / parseSearchFilters / activeFilters / filtersCacheKey');
// ---------- Bornes ----------
eq(periodMs('all'), 0, 'period all -> no bound');
expect(periodMs('hour') === 3600 * 1000, 'hour bound');
expect(periodMs('week') === 7 * 24 * 3600 * 1000, 'week bound');
eq(durationBounds('short'), { min: 1, max: 240 }, 'short duration bound');
eq(durationBounds('medium'), { min: 240, max: 1200 }, 'medium duration bound');
eq(durationBounds('long'), { min: 1200, max: Infinity }, 'long duration bound');
eq(durationBounds('all'), null, 'no duration bound');
logOk('periodMs / durationBounds');
// ---------- Lectures de métadonnées ----------
eq(itemDurationSec({ duration: 120 }), 120, 'numeric duration');
eq(itemDurationSec({ duration: 'PT1H2M' }), 0, 'ISO duration ignored (not a field format)');
eq(itemDurationSec({ duration: '1:02:03' }), 3723, 'hh:mm:ss duration parsed');
eq(itemDurationSec({}), 0, 'missing duration -> 0');
expect(itemPublishedTs({ publishedAt: new Date().toISOString() }) > 0, 'ISO publishedAt parsed');
expect(itemPublishedTs({ timestamp: 1700000000 }) === 1700000000000, 'yt-dlp seconds timestamps up-scaled');
eq(itemPublishedTs({}), 0, 'missing date -> 0');
logOk('itemDurationSec / itemPublishedTs');
// ---------- Post-filtrage ----------
const now = Date.now();
// `type: 'video'` par défaut = payload réaliste d'un provider qui expose le type.
const mk = (over) => ({
id: Math.random().toString(36).slice(2),
title: 'x',
type: 'video',
duration: 100,
publishedAt: new Date(now - 60 * 1000).toISOString(),
...over,
});
expect(matchesFilters(mk({ duration: 100 }), { duration: 'short' }), 'short keeps 100s');
expect(!matchesFilters(mk({ duration: 300 }), { duration: 'short' }), 'short drops 5min');
expect(matchesFilters(mk({ duration: 300 }), { duration: 'medium' }), 'medium keeps 5min');
expect(matchesFilters(mk({ duration: 3600 }), { duration: 'long' }), 'long keeps 1h');
expect(matchesFilters(mk({ duration: null }), { duration: 'short' }), 'missing duration is never dropped');
expect(
matchesFilters(mk({ publishedAt: new Date(now - 3 * 3600 * 1000).toISOString() }), { period: 'today' }),
'today keeps 3h ago',
);
expect(
!matchesFilters(mk({ publishedAt: new Date(now - 10 * 24 * 3600 * 1000).toISOString() }), { period: 'week' }),
'week drops 10 days ago',
);
expect(
matchesFilters(mk({ publishedAt: new Date(now - 3 * 24 * 3600 * 1000).toISOString() }), { period: 'week' }),
'week keeps 3 days ago',
);
expect(matchesFilters(mk({ publishedAt: null }), { period: 'today' }), 'missing date is never dropped');
expect(matchesFilters(mk({ isLive: true }), { type: 'live' }), 'live type keeps isLive items');
expect(!matchesFilters(mk({}), { type: 'live' }), 'live type drops regular videos');
expect(matchesFilters(mk({ isShort: true, duration: 40 }), { type: 'shorts' }), 'shorts keeps flagged items');
expect(matchesFilters(mk({ kind: 'clip' }), { type: 'shorts' }), 'shorts keeps clips');
// Item sans aucun signal de type : on ne peut pas conclure -> jamais écarté.
expect(matchesFilters(mk({ duration: 0, type: undefined }), { type: 'live' }), 'item without type signal is kept');
expect(matchesFilters(mk({ duration: 0, type: undefined }), { type: 'shorts' }), 'item without type signal is kept for shorts');
expect(
matchesFilters(mk({ duration: 500 }), { type: 'video' }),
'video type keeps plain videos',
);
expect(!matchesFilters(mk({ isLive: true }), { type: 'video' }), 'video type drops live items');
logOk('matchesFilters (durée / période / type)');
// ---------- applySearchFilters par provider ----------
const items = [
mk({ duration: 100, isShort: true }),
mk({ duration: 1800 }),
mk({ isLive: true, duration: 60 }),
mk({ duration: 0, publishedAt: null }), // métadonnées absentes : jamais écarté
];
eq(
applySearchFilters('yt', items, { duration: 'short' }).length,
3,
'yt post-filters duration (100s + 60s live kept, 1800s dropped, unknown kept)',
);
eq(
applySearchFilters('tw', items, { duration: 'short' }).length,
items.length,
'twitch does not post-filter duration (no reliable value)',
);
eq(
applySearchFilters('dm', items, { type: 'channel' }).length,
items.length,
'channel filter is not applied outside YouTube (would empty results)',
);
eq(
applySearchFilters('yt', items, { type: 'channel' }).length,
0,
'yt channel filter drops videos',
);
eq(
applySearchFilters('yt', items, {}).length,
items.length,
'no filter -> list untouched (same reference semantics)',
);
eq(
applySearchFilters('yt', [], { type: 'live' }),
[],
'empty list stays empty',
);
logOk('applySearchFilters (dimensions par provider)');
// ---------- Mapping providers ----------
eq(innertubeSearchFilters({ period: 'week' }), { upload_date: 'week' }, 'period -> upload_date');
eq(innertubeSearchFilters({ period: 'hour' }), {}, 'hour has no InnerTube enum (post-filter only)');
eq(innertubeSearchFilters({ type: 'shorts' }), { type: 'shorts' }, 'type shorts');
eq(innertubeSearchFilters({ type: 'live' }), { features: ['live'] }, 'live is a feature, not a SearchType');
eq(innertubeSearchFilters({ type: 'channel' }), { type: 'channel' }, 'type channel');
eq(
innertubeSearchFilters({ duration: 'short' }),
{ duration: 'UNDER_THREE_MINS' },
'short -> UNDER_THREE_MINS proto enum',
);
eq(
innertubeSearchFilters({ duration: 'medium' }),
{ duration: 'THREE_TO_TWENTY_MINS' },
'medium -> THREE_TO_TWENTY_MINS proto enum',
);
eq(
innertubeSearchFilters({ duration: 'long' }),
{ duration: 'OVER_TWENTY_MINS' },
'long -> OVER_TWENTY_MINS proto enum',
);
eq(innertubeSearchFilters({}), {}, 'no filters -> empty object');
eq(apiSearchParams({ type: 'video' }), { type: 'video' }, 'api type video');
eq(apiSearchParams({ duration: 'long' }), { videoDuration: 'long' }, 'api videoDuration');
eq(apiSearchParams({ type: 'shorts' }), {}, 'api has no shorts type (post-filter)');
expect(
typeof apiSearchParams({ period: 'today' }).publishedAfter === 'string',
'api publishedAfter ISO date',
);
logOk('innertubeSearchFilters / apiSearchParams');
console.log('\nAll search-filters tests passed.');
+27
View File
@@ -155,6 +155,33 @@ function scenario(name, fn) {
// 2b) Invalid providers fall back to all
await scenario('invalid providers param falls back to the full registry', async () => validateProvidersFallback());
// 2c) Filtres de recherche : ils sont renvoyés tels quels, valeurs inconnues ignorées
await scenario('search filters are echoed back and unknown values ignored', async () => {
const res = await fetch(`${baseUrl}/api/search?q=test&providers=yt,dm&pageSize=3&type=live&duration=short&period=week&sort=date`);
assert(res.ok, `search responds 200 (got ${res.status})`);
const body = await res.json();
assert(body.filters && body.filters.type === 'live', `type filter echoed (got ${JSON.stringify(body.filters)})`);
assert(body.filters.duration === 'short', 'duration filter echoed');
assert(body.filters.period === 'week', 'period filter echoed');
assert(body.sort === 'date', 'sort comes from the filters');
// Valeurs inconnues : ignorées, jamais 400.
const bad = await fetch(`${baseUrl}/api/search?q=test&providers=dm&type=plaid&period=yesterday&duration=epic`);
assert(bad.ok, `unknown filter values do not fail (got ${bad.status})`);
const badBody = await bad.json();
assert(!badBody.filters || Object.keys(badBody.filters).length === 0,
`unknown filters are dropped (got ${JSON.stringify(badBody.filters)})`);
assert(badBody.sort === 'relevance', 'sort falls back to relevance');
});
// 2d) Un filtre ne doit jamais renvoyer d'erreur 500 (post-filtrage sûr)
await scenario('filters never break the fan-out (200 + groups)', async () => {
const res = await fetch(`${baseUrl}/api/search?q=test&pageSize=3&period=today&duration=long`);
assert(res.ok, `search responds 200 with filters (got ${res.status})`);
const body = await res.json();
assert(body.groups && typeof body.groups === 'object', 'groups present with filters');
});
// 3) Preference: defaultProviders persisted and returned after re-login
await scenario('default providers preference round-trip (persisted per user)', async () => {
await patchPreferences(token, { defaultProviders: ['ru', 'od'] });
+46 -3
View File
@@ -1,6 +1,6 @@
// Step 18 — tests offline InnerTube (AUCUN réseau : mappers purs + modes).
import assert from 'node:assert/strict';
import { mapVideoNode, mapLockupView, mapNodes, mapCaptionTracks, parseViewsText, parseDurationLabel } from '../providers/youtube-innertube.mjs';
import { mapVideoNode, mapLockupView, mapNodes, mapShelfNodes, mapCaptionTracks, parseViewsText, parseDurationLabel } from '../providers/youtube-innertube.mjs';
import { pickTrack, orderedTracks } from '../transcript.mjs';
import { getSearchMode, YT_SEARCH_MODES } from '../providers/youtube-common.mjs';
@@ -96,7 +96,50 @@ function ok(msg) { console.log(`✓ ${msg}`); }
ok('mapLockupView watch-next');
}
// 5) modes innertube connus
// 5) mapShelfNodes + ShortsLockupView : les Shorts d'une recherche « Shorts »
// n'ont ni `id` ni `title` classiques (endpoint + overlay_metadata), et
// vivent dans un GridShelfView -> sans ce complément la recherche renvoie 0.
{
// Forme réelle observée (youtubei.js v18, ShortsLockupView)
const shortsNode = (id) => ({
type: 'ShortsLockupView',
entity_id: `shorts-shelf-item-${id}`,
on_tap_endpoint: { name: 'reelWatchEndpoint', payload: { videoId: id } },
overlay_metadata: {
primary_text: { text: `Short ${id}` },
secondary_text: { text: '219 M de vues' },
},
thumbnail: [{ url: 'http://t/low.jpg', width: 100 }, { url: 'http://t/high.jpg', width: 1080 }],
});
const mapped = mapVideoNode(shortsNode('s1'));
assert.equal(mapped.id, 's1', 'l\'id du Short est lu dans on_tap_endpoint');
assert.equal(mapped.title, 'Short s1', 'le titre vient de overlay_metadata');
assert.equal(mapped.isShort, true, 'un Short est marqué isShort');
assert.equal(mapped.views, 219000000, 'les vues viennent de la 2e ligne d\'overlay');
assert.equal(mapped.thumbnail, 'http://t/high.jpg', 'la plus grande vignette est retenue');
// Forme « brute » (avant parsing youtubei.js) également gérée
assert.equal(mapVideoNode({
type: 'ShortsLockupView',
on_tap: { innertubeCommand: { reelWatchEndpoint: { videoId: 'raw1' } } },
overlay_metadata: { primary_text: { text: 'Raw' } },
}).id, 'raw1', 'forme brute (on_tap) gérée');
const feed = {
results: [
{ type: 'Video', id: 'regular1', title: { text: 'Vidéo normale' } },
{ type: 'GridShelfView', contents: [shortsNode('s1'), shortsNode('s2'), { type: 'ChipCloud' }] },
],
};
const shelf = mapShelfNodes(feed);
assert.equal(shelf.length, 2, 'les 2 Shorts du shelf sont mappés');
assert.deepEqual(shelf.map((s) => s.id).sort(), ['s1', 's2']);
assert.equal(mapShelfNodes({ results: [{ type: 'Video', id: 'v' }] }).length, 0, 'hors shelf, rien à faire');
assert.deepEqual(mapShelfNodes(null), []);
assert.deepEqual(mapShelfNodes({}), []);
ok('mapShelfNodes + ShortsLockupView (Shorts dans un shelf)');
}
// 6) modes innertube connus
{
assert.ok(YT_SEARCH_MODES.includes('innertube-first'));
assert.ok(YT_SEARCH_MODES.includes('innertube-only'));
@@ -107,7 +150,7 @@ function ok(msg) { console.log(`✓ ${msg}`); }
ok('modes innertube-first/only');
}
// 6) mapCaptionTracks : forme InnerTube -> pseudo dump yt-dlp, interop pickTrack
// 7) mapCaptionTracks : forme InnerTube -> pseudo dump yt-dlp, interop pickTrack
{
const cap = mapCaptionTracks([
{ base_url: 'https://x/t?caps=asr&x=1', language_code: 'en', kind: 'asr', name: { text: 'English (auto-generated)' } },