feat(youtube): InnerTube-first search, transcripts and watch-next related (Steps 15-18)
- InnerTube layer via pinned youtubei.js 18.1.0 (no quota, no key): search with merged continuations (unlimited pages), watch-next related with LockupView mapping, caption-track discovery - 3-layer dispatcher (YT_SEARCH_MODE, default innertube-first): innertube -> yt-dlp scrape -> official API, graceful errors.yt - Robust yt-dlp binary resolution (YT_DLP_PATH > PATH > bundled) with systematic API fallback (fixes spawn ENOENT in UI) - Transcript: InnerTube caption discovery (YT_TRANSCRIPT_SOURCE), reusing pickTrack/orderedTracks/parseTrackText; yt-dlp fallback kept - Watch: sidebar uses real watch-next related[] (/api/details), title-search fallback for other providers - Cache: memory LRU + SQLite (youtube_search_cache, youtube_metrics), never persist empty pages; /healthz observability; /api/trending - Includes pending Step 15/16 leftovers in same files (suggest, test scripts); unrelated provider adapters left uncommitted
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
import { Injectable, inject } from '@angular/core';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { Observable, of } from 'rxjs';
|
||||
import { tap } from 'rxjs/operators';
|
||||
import { dedupeSort } from './suggest.util';
|
||||
|
||||
export interface SuggestResponse {
|
||||
q: string;
|
||||
groups: Record<string, string[]>;
|
||||
}
|
||||
|
||||
interface SuggestCacheEntry {
|
||||
t: number;
|
||||
data: SuggestResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* Step 15 — Query typeahead service.
|
||||
*
|
||||
* Contract: no network while `q.trim().length < 2`. Callers debounce
|
||||
* (250-300ms) + `switchMap` (in-flight cancellation). Responses are cached
|
||||
* 5 minutes, deduped and sorted per provider group.
|
||||
*/
|
||||
@Injectable({ providedIn: 'root' })
|
||||
export class SuggestService {
|
||||
private http = inject(HttpClient);
|
||||
private cache = new Map<string, SuggestCacheEntry>();
|
||||
private cacheTtlMs = 5 * 60 * 1000;
|
||||
|
||||
private apiBase(): string {
|
||||
try {
|
||||
const port = window?.location?.port || '';
|
||||
return port && port !== '4000' ? '/proxy/api' : '/api';
|
||||
} catch {
|
||||
return '/api';
|
||||
}
|
||||
}
|
||||
|
||||
/** Single suggest round-trip (no debounce here — the caller owns it). */
|
||||
suggestOnce(q: string, providers?: string[] | 'all', limit = 10): Observable<SuggestResponse> {
|
||||
const query = String(q ?? '').trim();
|
||||
if (query.length < 2) return of({ q: query, groups: {} });
|
||||
const prov = Array.isArray(providers) && providers.length > 0 ? [...providers].sort().join(',') : 'all';
|
||||
const key = `${prov}|${query.toLowerCase()}|${limit}`;
|
||||
const now = Date.now();
|
||||
const hit = this.cache.get(key);
|
||||
if (hit && now - hit.t < this.cacheTtlMs) return of(hit.data);
|
||||
|
||||
const params = new URLSearchParams({ q: query, limit: String(limit) });
|
||||
if (Array.isArray(providers) && providers.length > 0) params.set('providers', providers.join(','));
|
||||
return this.http.get<SuggestResponse>(`${this.apiBase()}/search/suggest?${params.toString()}`).pipe(
|
||||
tap((res) => {
|
||||
try {
|
||||
const groups: Record<string, string[]> = {};
|
||||
for (const [pid, list] of Object.entries(res?.groups || {})) {
|
||||
groups[pid] = dedupeSort(Array.isArray(list) ? list : [], query).slice(0, limit);
|
||||
}
|
||||
const clean: SuggestResponse = { q: res?.q ?? query, groups };
|
||||
if (this.cache.size > 200) this.cache.clear();
|
||||
this.cache.set(key, { t: Date.now(), data: clean });
|
||||
// Mutate in place so subscribers see the cleaned payload
|
||||
res.q = clean.q;
|
||||
res.groups = clean.groups;
|
||||
} catch {}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
clearCache(): void {
|
||||
this.cache.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import 'zone.js/node';
|
||||
import '@angular/compiler'; // JIT needed: decorators are evaluated at import time under ts-node
|
||||
import { firstValueFrom, of } from 'rxjs';
|
||||
import { Injector, runInInjectionContext } from '@angular/core';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { SuggestService } from './suggest.service';
|
||||
import { dedupeSort, mergeGroups, highlightParts } from './suggest.util';
|
||||
|
||||
/**
|
||||
* Step 15 — Unit tests for query typeahead (parsing / dedup / min-length guard).
|
||||
*
|
||||
* Same harness as Step 11: ts-node isolated transpilation, manual injection
|
||||
* with a minimal HttpClient fake returning real Observables.
|
||||
* Run with: npm run test:suggest
|
||||
*/
|
||||
|
||||
function assertEqual(actual: unknown, expected: unknown, message: string): void {
|
||||
const a = JSON.stringify(actual);
|
||||
const b = JSON.stringify(expected);
|
||||
if (a !== b) {
|
||||
throw new Error(`${message} (expected ${b}, got ${a})`);
|
||||
}
|
||||
}
|
||||
|
||||
function assert(condition: unknown, message: string): void {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
function logOk(msg: string): void {
|
||||
console.log(`✓ ${msg}`);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
// --- Pure helpers: dedupe + sort ---
|
||||
assertEqual(
|
||||
dedupeSort(['Tutoriel Angular', 'tutoriel angular ', ' ', 'Tutoriel Android'], 'tutoriel an'),
|
||||
['Tutoriel Android', 'Tutoriel Angular'],
|
||||
'dedupeSort dedupes case-insensitively and sorts prefix-first then alpha',
|
||||
);
|
||||
logOk('dedupeSort dedup + tri');
|
||||
|
||||
assertEqual(dedupeSort(['b', 'a', 'c']), ['a', 'b', 'c'], 'dedupeSort sorts alphabetically without query');
|
||||
logOk('dedupeSort alpha fallback');
|
||||
|
||||
// --- Pure helpers: merge groups ---
|
||||
const merged = mergeGroups({ dm: ['Tutoriel android'], yt: ['Tutoriel Angular', 'tutoriel android'], xx: ['zz'] });
|
||||
assertEqual(
|
||||
merged,
|
||||
[
|
||||
{ text: 'Tutoriel Angular', provider: 'yt' },
|
||||
{ text: 'tutoriel android', provider: 'yt' },
|
||||
{ text: 'zz', provider: 'xx' },
|
||||
],
|
||||
'mergeGroups respects registry order, dedupes across providers, unknowns last',
|
||||
);
|
||||
logOk('mergeGroups order + dedup');
|
||||
|
||||
// --- Pure helpers: highlight ---
|
||||
assertEqual(
|
||||
highlightParts('Tutoriel Angular', 'toriel an'),
|
||||
{ pre: 'Tu', match: 'toriel An', post: 'gular' },
|
||||
'highlightParts splits on first case-insensitive occurrence',
|
||||
);
|
||||
assertEqual(
|
||||
highlightParts('Bonjour', 'zzz'),
|
||||
{ pre: 'Bonjour', match: '', post: '' },
|
||||
'highlightParts without occurrence returns plain text',
|
||||
);
|
||||
logOk('highlightParts');
|
||||
|
||||
// --- Service: min length guard (no network below 2 chars) ---
|
||||
let httpCalls = 0;
|
||||
const httpFake = {
|
||||
get: (url: string) => {
|
||||
httpCalls += 1;
|
||||
assert(url.includes('/search/suggest'), `suggest hits /api/search/suggest (got ${url})`);
|
||||
return of({ q: 'angular', groups: { yt: ['angular tutorial', 'Angular'], dm: ['angular tutorial'] } });
|
||||
},
|
||||
};
|
||||
const injector = Injector.create([{ provide: HttpClient, useValue: httpFake }]);
|
||||
const svc: SuggestService = runInInjectionContext(injector, () => new SuggestService());
|
||||
|
||||
const short = await firstValueFrom(svc.suggestOnce('a'));
|
||||
assertEqual(short, { q: 'a', groups: {} }, 'q < 2 chars returns empty without network');
|
||||
assertEqual(httpCalls, 0, 'no HTTP call for short queries');
|
||||
logOk('suggestOnce min-length guard (no network)');
|
||||
|
||||
// --- Service: dedup + cache ---
|
||||
const first = await firstValueFrom(svc.suggestOnce('angular', ['yt', 'dm']));
|
||||
assertEqual(httpCalls, 1, 'one HTTP call for a fresh query');
|
||||
assertEqual(first.groups['yt'], ['Angular', 'angular tutorial'], 'per-provider groups deduped + sorted');
|
||||
const second = await firstValueFrom(svc.suggestOnce('angular', ['dm', 'yt']));
|
||||
assertEqual(httpCalls, 1, 'second identical query served from cache (provider order-insensitive)');
|
||||
assert(second === first || JSON.stringify(second) === JSON.stringify(first), 'cached payload matches');
|
||||
logOk('suggestOnce dedup + cache 5 min');
|
||||
|
||||
console.log('\nAll suggest tests passed.');
|
||||
})().catch((e) => {
|
||||
console.error(e instanceof Error ? e.stack : String(e));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Step 15 — Pure helpers for query typeahead (testable without Angular).
|
||||
*/
|
||||
|
||||
/** Normalize a suggestion string for dedup comparisons. */
|
||||
export function normSuggest(s: string): string {
|
||||
return String(s ?? '').trim().toLowerCase().replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Deduplicate (case-insensitive, first occurrence wins) then sort:
|
||||
* items starting with `q` come first, alphabetical within each group.
|
||||
*/
|
||||
export function dedupeSort(items: string[], q = ''): string[] {
|
||||
const seen = new Set<string>();
|
||||
const clean: string[] = [];
|
||||
for (const raw of items || []) {
|
||||
const t = String(raw ?? '').trim();
|
||||
if (!t) continue;
|
||||
const k = normSuggest(t);
|
||||
if (seen.has(k)) continue;
|
||||
seen.add(k);
|
||||
clean.push(t);
|
||||
}
|
||||
const nq = normSuggest(q);
|
||||
clean.sort((a, b) => {
|
||||
const na = normSuggest(a);
|
||||
const nb = normSuggest(b);
|
||||
const pa = nq ? (na.startsWith(nq) ? 0 : 1) : 0;
|
||||
const pb = nq ? (nb.startsWith(nq) ? 0 : 1) : 0;
|
||||
if (pa !== pb) return pa - pb;
|
||||
return na < nb ? -1 : na > nb ? 1 : 0;
|
||||
});
|
||||
return clean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flatten provider groups into a single ordered list (registry order),
|
||||
* deduped case-insensitively. Unknown provider keys go last (insertion order).
|
||||
*/
|
||||
export function mergeGroups(
|
||||
groups: Record<string, string[]> | null | undefined,
|
||||
order: string[] = ['yt', 'dm', 'tw', 'pt', 'od', 'ru'],
|
||||
): Array<{ text: string; provider: string }> {
|
||||
const out: Array<{ text: string; provider: string }> = [];
|
||||
const seen = new Set<string>();
|
||||
const push = (pid: string, list: string[] | undefined) => {
|
||||
for (const raw of list || []) {
|
||||
const t = String(raw ?? '').trim();
|
||||
if (!t) continue;
|
||||
const k = normSuggest(t);
|
||||
if (seen.has(k)) continue;
|
||||
seen.add(k);
|
||||
out.push({ text: t, provider: pid });
|
||||
}
|
||||
};
|
||||
for (const pid of order) push(pid, groups?.[pid]);
|
||||
for (const pid of Object.keys(groups || {})) {
|
||||
if (!order.includes(pid)) push(pid, (groups as Record<string, string[]>)[pid]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Split `text` on the first case-insensitive occurrence of `q` for <mark> highlighting. */
|
||||
export function highlightParts(text: string, q: string): { pre: string; match: string; post: string } {
|
||||
const t = String(text ?? '');
|
||||
const needle = String(q ?? '').trim();
|
||||
if (!needle) return { pre: t, match: '', post: '' };
|
||||
const idx = t.toLowerCase().indexOf(needle.toLowerCase());
|
||||
if (idx === -1) return { pre: t, match: '', post: '' };
|
||||
return { pre: t.slice(0, idx), match: t.slice(idx, idx + needle.length), post: t.slice(idx + needle.length) };
|
||||
}
|
||||
Reference in New Issue
Block a user