fix(sync): WL diagnostic, import-all paginé + totaux, sync auto historique InnerTube

- Watch Later: youtube_no_channel / watchlater_not_found avec hint
  actionnable + log cles relatedPlaylists (sans PII)
- Preview Google: totaux compte (subscriptionTotal/likeTotal), UI
  'X / N, l'import prend tout'; import-all paginé (subs<=1000,
  likes<=500) au lieu de 50/25
- Sync auto historique: GET /oauth/google/yt-history (browse FEhistory
  authentifie, continuations, <=500), bouton Sync auto dans carte
  Takeout, import via lots takeout existants
This commit is contained in:
2026-09-26 22:25:50 -04:00
parent 1c013452fa
commit 050cea3a92
5 changed files with 260 additions and 33 deletions
+59 -8
View File
@@ -105,7 +105,7 @@ import {
oauthStatus, buildAuthUrl, createOAuthState, consumeOAuthState, exchangeCode,
refreshAccessToken, redirectUriFor, fetchGoogleProfile, fetchGoogleSubscriptions,
fetchGoogleLiked, fetchTwitchProfile, fetchTwitchFollows, hasGoogleWriteScope,
fetchGoogleWatchLater, pushGoogleWatchLater,
fetchGoogleWatchLater, pushGoogleWatchLater, fetchInnerTubeHistory,
} from './oauth.mjs';
import { getSearchMode, getYtDlpBin, hasCookiesFile, metricsSnapshot } from './providers/youtube-common.mjs';
import { ytScrapeCacheStats } from './providers/youtube.mjs';
@@ -564,11 +564,20 @@ r.get('/oauth/:provider/preview', authMiddlewareCookieAware, oauthLimiter, async
const provider = requireOAuthProvider(req.params.provider);
const conn = await freshOAuthToken(req.user.id, provider);
if (provider === 'google') {
const [subscriptions, likes] = await Promise.all([
fetchGoogleSubscriptions(conn.accessToken, 50).catch(() => []),
fetchGoogleLiked(conn.accessToken, 25).catch(() => []),
const [subsRes, likesRes] = await Promise.all([
fetchGoogleSubscriptions(conn.accessToken, 50).catch(() => ({ items: [], total: 0 })),
fetchGoogleLiked(conn.accessToken, 25).catch(() => ({ items: [], total: 0 })),
]);
return res.json({ provider, subscriptions, likes, subscriptionCount: subscriptions.length, likeCount: likes.length });
return res.json({
provider,
subscriptions: subsRes.items,
likes: likesRes.items,
subscriptionCount: subsRes.items.length,
likeCount: likesRes.items.length,
// Totaux du compte (l'aperçu n'affiche que les 50/25 premiers, l'import prend tout).
subscriptionTotal: subsRes.total,
likeTotal: likesRes.total,
});
}
const profile = await fetchTwitchProfile(conn.accessToken).catch(() => ({ id: conn.externalUserId || '' }));
const follows = await fetchTwitchFollows(conn.accessToken, profile?.id || conn.externalUserId || '', 100);
@@ -594,7 +603,8 @@ r.post('/oauth/:provider/import', authMiddlewareCookieAware, oauthLimiter, async
if (provider === 'google') {
if (wantSubs) {
const subs = await fetchGoogleSubscriptions(conn.accessToken, 50);
// Import complet paginé (l'aperçu n'en montre que 50).
const { items: subs } = await fetchGoogleSubscriptions(conn.accessToken, 1000);
for (const s of subs) {
try {
const row = upsertChannelRow({
@@ -608,7 +618,8 @@ r.post('/oauth/:provider/import', authMiddlewareCookieAware, oauthLimiter, async
}
}
if (wantLikes) {
const likes = await fetchGoogleLiked(conn.accessToken, 25);
// Import complet paginé (l'aperçu n'en montre que 25).
const { items: likes } = await fetchGoogleLiked(conn.accessToken, 500);
for (const v of likes) {
try {
likeVideo({ userId: req.user.id, provider: 'youtube', videoId: v.videoId, title: v.title, thumbnail: v.thumbnail });
@@ -655,7 +666,47 @@ r.get('/oauth/google/watchlater', authMiddlewareCookieAware, oauthLimiter, async
writeGranted: hasGoogleWriteScope(conn.scopes),
});
} catch (error) {
return res.status(error?.status || 502).json({ error: error?.message || 'watchlater_fetch_failed' });
const out = { error: error?.message || 'watchlater_fetch_failed' };
if (error?.hint) out.hint = error.hint;
return res.status(error?.status || 502).json(out);
}
});
// -------------------- Google : historique auto via InnerTube authentifié --------------------
// L'API Data v3 n'expose pas l'historique : on lit youtubei/v1/browse
// (FEhistory) avec le Bearer OAuth de l'utilisateur (mécanisme SmartTube).
// Le client réutilise ensuite POST /user/history/takeout pour l'importer.
function innertubeApiKey() {
const single = String(process.env.YOUTUBE_API_KEY || '').trim();
if (single && !single.includes(',')) return single;
const csv = String(process.env.YOUTUBE_API_KEYS || '').trim();
try {
if (csv.startsWith('[')) {
const arr = JSON.parse(csv);
if (Array.isArray(arr) && arr[0]) return String(arr[0]);
}
} catch {}
if (csv) {
const first = csv.split(',').map((s) => s.trim()).filter(Boolean)[0];
if (first) return first;
}
if (single) return single.split(',').map((s) => s.trim()).filter(Boolean)[0] || '';
return '';
}
r.get('/oauth/google/yt-history', authMiddlewareCookieAware, oauthLimiter, async (req, res) => {
try {
const conn = await freshOAuthToken(req.user.id, 'google');
const max = Math.min(500, Math.max(1, Number(req.query.max || 200)));
const { items, hasMore } = await fetchInnerTubeHistory(conn.accessToken, innertubeApiKey(), max);
return res.json({ items, count: items.length, hasMore });
} catch (error) {
const out = { error: error?.message || 'ythistory_fetch_failed' };
if (error?.status === 401) {
out.hint = 'Session Google expirée : déconnectez puis reconnectez Google.';
}
return res.status(error?.status || 502).json(out);
}
});
+147 -20
View File
@@ -254,8 +254,9 @@ export async function fetchGoogleProfile(accessToken) {
};
}
export async function fetchGoogleSubscriptions(accessToken, max = 50) {
export async function fetchGoogleSubscriptions(accessToken, max = 1000) {
const out = [];
let total = 0;
let pageToken = '';
while (out.length < max) {
const qs = new URLSearchParams({
@@ -266,6 +267,7 @@ export async function fetchGoogleSubscriptions(accessToken, max = 50) {
});
if (pageToken) qs.set('pageToken', pageToken);
const data = await getJson(`https://www.googleapis.com/youtube/v3/subscriptions?${qs.toString()}`, accessToken);
if (!total && typeof data?.pageInfo?.totalResults === 'number') total = data.pageInfo.totalResults;
for (const item of Array.isArray(data?.items) ? data.items : []) {
const sn = item?.snippet || {};
const channelId = sn?.resourceId?.channelId || '';
@@ -282,29 +284,37 @@ export async function fetchGoogleSubscriptions(accessToken, max = 50) {
pageToken = data?.nextPageToken || '';
if (!pageToken) break;
}
return out;
return { items: out, total: total || out.length };
}
export async function fetchGoogleLiked(accessToken, max = 25) {
const qs = new URLSearchParams({
part: 'snippet,contentDetails',
myRating: 'like',
maxResults: String(Math.min(50, max)),
});
const data = await getJson(`https://www.googleapis.com/youtube/v3/videos?${qs.toString()}`, accessToken);
export async function fetchGoogleLiked(accessToken, max = 500) {
const out = [];
for (const item of Array.isArray(data?.items) ? data.items : []) {
const id = item?.id || '';
const sn = item?.snippet || {};
if (!id) continue;
out.push({
provider: 'youtube',
videoId: String(id),
title: sn?.title || '',
thumbnail: sn?.thumbnails?.medium?.url || sn?.thumbnails?.default?.url || '',
let total = 0;
let pageToken = '';
while (out.length < max) {
const qs = new URLSearchParams({
part: 'snippet,contentDetails',
myRating: 'like',
maxResults: String(Math.min(50, max - out.length)),
});
if (pageToken) qs.set('pageToken', pageToken);
const data = await getJson(`https://www.googleapis.com/youtube/v3/videos?${qs.toString()}`, accessToken);
if (!total && typeof data?.pageInfo?.totalResults === 'number') total = data.pageInfo.totalResults;
for (const item of Array.isArray(data?.items) ? data.items : []) {
const id = item?.id || '';
const sn = item?.snippet || {};
if (!id) continue;
out.push({
provider: 'youtube',
videoId: String(id),
title: sn?.title || '',
thumbnail: sn?.thumbnails?.medium?.url || sn?.thumbnails?.default?.url || '',
});
}
pageToken = data?.nextPageToken || '';
if (!pageToken) break;
}
return out;
return { items: out, total: total || out.length };
}
// -------------------- Google : Watch Later --------------------
@@ -337,8 +347,23 @@ async function googleApiPost(accessToken, path, body) {
export async function fetchGoogleWatchLaterId(accessToken) {
const data = await googleApiGet(accessToken, 'channels', { part: 'contentDetails', mine: 'true' });
const item = Array.isArray(data?.items) ? data.items[0] : null;
if (!item) {
throw Object.assign(new Error('youtube_no_channel'), {
status: 502,
hint: 'Ce compte Google n’a pas de chaîne YouTube : créez-en une sur youtube.com pour utiliser Regarder plus tard.',
});
}
const id = item?.contentDetails?.relatedPlaylists?.watchLater || '';
if (!id) throw Object.assign(new Error('watchlater_not_found'), { status: 502 });
if (!id) {
// Diagnostic sans PII : quelles playlists liées YouTube expose-t-il ?
try {
console.warn('[oauth] watchLater absent, relatedPlaylists =', Object.keys(item?.contentDetails?.relatedPlaylists || {}).join(',') || '(vide)');
} catch {}
throw Object.assign(new Error('watchlater_not_found'), {
status: 502,
hint: 'YouTube ne renvoie pas de playlist « Regarder plus tard » pour ce compte (compte de marque sans chaîne principale ?). Les abonnements et favoris restent importables.',
});
}
return String(id);
}
@@ -384,6 +409,108 @@ export async function pushGoogleWatchLater(accessToken, videoId) {
return { playlistItemId: data?.id || null };
}
// -------------------- Historique YouTube via InnerTube authentifié --------------------
// L'API Data v3 n'expose pas l'historique : on passe par youtubei/v1/browse
// (browseId FEhistory) avec le Bearer OAuth de l'utilisateur — même mécanisme
// que SmartTube. Lecture seule, pas de scope supplémentaire.
function pickThumb(thumbnails) {
const list = Array.isArray(thumbnails) ? thumbnails : [];
let best = null;
for (const t of list) {
if (!t?.url) continue;
if (!best || Number(t.width || 0) > Number(best.width || 0)) best = t;
}
return best?.url || '';
}
function runsText(runs) {
try {
return (Array.isArray(runs) ? runs : []).map((r) => r?.text || '').join('');
} catch { return ''; }
}
function extractHistoryEntries(node, out) {
if (!node || typeof node !== 'object') return;
if (Array.isArray(node)) {
for (const e of node) extractHistoryEntries(e, out);
return;
}
if (node.videoRenderer?.videoId) {
const vr = node.videoRenderer;
out.push({
provider: 'youtube',
videoId: String(vr.videoId),
title: runsText(vr.title?.runs) || String(vr.title?.simpleText || vr.videoId),
thumbnail: pickThumb(vr.thumbnail?.thumbnails),
watchedAt: new Date().toISOString(),
});
return;
}
for (const v of Object.values(node)) {
if (v && typeof v === 'object') extractHistoryEntries(v, out);
}
}
function findHistoryContinuation(data) {
try {
const tabs = data?.contents?.singleColumnBrowseResultsRenderer?.tabs || [];
const stack = [...tabs];
while (stack.length) {
const n = stack.pop();
if (!n || typeof n !== 'object') continue;
if (Array.isArray(n)) { stack.push(...n); continue; }
const token = n?.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token;
if (token) return String(token);
for (const v of Object.values(n)) {
if (v && typeof v === 'object') stack.push(v);
}
}
} catch {}
return null;
}
export async function fetchInnerTubeHistory(accessToken, apiKey, max = 200) {
const key = String(apiKey || '').trim();
if (!key) throw Object.assign(new Error('youtube_api_key_unavailable'), { status: 503 });
const out = [];
const seen = new Set();
let continuation = null;
let pages = 0;
while (out.length < max && pages < 10) {
pages++;
const body = continuation
? { context: { client: { clientName: 'WEB', clientVersion: '2.20260101.00.00', hl: 'fr', gl: 'FR' } }, continuation }
: { context: { client: { clientName: 'WEB', clientVersion: '2.20260101.00.00', hl: 'fr', gl: 'FR' } }, browseId: 'FEhistory' };
const resp = await fetch(`https://www.youtube.com/youtubei/v1/browse?key=${encodeURIComponent(key)}&prettyPrint=false`, {
method: 'POST',
headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(20000),
});
const data = await resp.json().catch(() => ({}));
if (!resp.ok) {
const err = new Error(`innertube_history_failed_${resp.status}`);
err.status = resp.status === 401 ? 401 : 502;
throw err;
}
const batch = [];
const root = continuation
? (data?.onResponseReceivedActions || data?.continuationContents)
: data;
extractHistoryEntries(root, batch);
for (const v of batch) {
if (seen.has(v.videoId)) continue;
seen.add(v.videoId);
out.push(v);
if (out.length >= max) break;
}
continuation = findHistoryContinuation(data);
if (!continuation) break;
}
return { items: out.slice(0, max), hasMore: Boolean(continuation) };
}
// -------------------- Twitch --------------------
function twitchClientId() {
@@ -69,7 +69,8 @@
<div *ngIf="errors()[p.id]" class="mb-2 rounded-lg border border-red-700 bg-red-900/30 p-2 text-xs text-red-200">{{ errors()[p.id] }}</div>
<div *ngIf="previews()[p.id] as pv" class="mb-2 text-sm">
<p class="text-slate-300">Abonnements : <strong>{{ pv.subscriptionCount }}</strong><span *ngIf="p.id === 'google'"> • Favoris : <strong>{{ pv.likeCount }}</strong></span></p>
<p class="text-slate-300">Abonnements : <strong>{{ pv.subscriptionCount }}</strong><span *ngIf="pv.subscriptionTotal && pv.subscriptionTotal > pv.subscriptionCount"> / {{ pv.subscriptionTotal }}</span><span *ngIf="p.id === 'google'"> • Favoris : <strong>{{ pv.likeCount }}</strong><span *ngIf="pv.likeTotal && pv.likeTotal > pv.likeCount"> / {{ pv.likeTotal }}</span></span></p>
<p *ngIf="p.id === 'google' && ((pv.subscriptionTotal && pv.subscriptionTotal > pv.subscriptionCount) || (pv.likeTotal && pv.likeTotal > pv.likeCount))" class="text-xs text-slate-500">Aperçu limité — le bouton Importer prend tout le compte.</p>
<ul class="mt-2 max-h-48 space-y-1 overflow-auto text-xs text-slate-400">
<li *ngFor="let s of pv.subscriptions.slice(0, 20)" class="flex items-center gap-2">
<img *ngIf="s.avatarUrl" [src]="s.avatarUrl" alt="" class="h-5 w-5 rounded-full object-cover" />
@@ -92,11 +93,15 @@
<div *ngIf="!loading()" class="mt-4 rounded-xl border border-slate-800 bg-slate-900/60 p-4">
<h3 class="text-lg font-semibold text-slate-100"><span class="mr-2">🕘</span>Historique YouTube (Takeout)</h3>
<p class="mt-1 text-sm text-slate-400">
Google n’expose pas l’historique par API : exportez-le via
<code>takeout.google.com → YouTube → watch-history.json</code>, puis importez le fichier ici.
Parsé <strong class="text-slate-200">localement dans votre navigateur</strong>, sans doublons.
Google n’expose pas l’historique par API Data : <strong class="text-slate-200">sync auto</strong> via votre compte
connecté, ou export manuel via <code>takeout.google.com → YouTube → watch-history.json</code>.
Fichier parsé <strong class="text-slate-200">localement dans votre navigateur</strong>, sans doublons.
</p>
<div class="mt-3 flex flex-wrap items-center gap-2">
<button *ngIf="isConnected('google')" type="button" (click)="syncAutoHistory()" [disabled]="ytAutoLoading() || takeoutImporting()"
class="rounded-md bg-red-600 px-3 py-1.5 text-sm font-semibold text-white transition hover:bg-red-500 disabled:opacity-40">
{{ ytAutoLoading() ? 'Sync…' : '⚡ Sync auto (compte Google)' }}
</button>
<label class="cursor-pointer rounded-md bg-slate-700 px-3 py-1.5 text-sm text-white transition hover:bg-slate-600">
Choisir watch-history.json
<input type="file" accept=".json,application/json" class="hidden" (change)="onTakeoutFile($event)" />
@@ -106,6 +111,7 @@
{{ takeoutImporting() ? 'Import…' : 'Importer dans l’historique' }}
</button>
</div>
<div *ngIf="ytAutoError()" class="mt-2 rounded-lg border border-red-700 bg-red-900/30 p-2 text-xs text-red-200">{{ ytAutoError() }}</div>
<div *ngIf="takeoutParsing()" class="mt-2 text-sm text-slate-400">Analyse du fichier…</div>
<div *ngIf="takeoutItems().length && takeoutRange() as rg" class="mt-2 text-sm text-slate-300">
{{ takeoutItems().length }} vidéo(s) — du {{ rg.from }} au {{ rg.to }}<span *ngIf="takeoutFileName()"> ({{ takeoutFileName() }})</span>
@@ -129,6 +129,39 @@ export class ImportComponent implements OnInit {
takeoutProgress = signal<{ done: number; total: number } | null>(null);
takeoutResult = signal<{ imported: number; skipped: number } | null>(null);
takeoutError = signal<string | null>(null);
ytAutoLoading = signal(false);
ytAutoError = signal<string | null>(null);
/** Sync auto : historique via InnerTube authentifié, puis import par lots. */
async syncAutoHistory(): Promise<void> {
if (!this.isConnected('google') || this.ytAutoLoading() || this.takeoutImporting()) return;
this.ytAutoLoading.set(true);
this.ytAutoError.set(null);
this.takeoutError.set(null);
this.takeoutResult.set(null);
this.takeoutProgress.set(null);
try {
const res = await firstValueFrom(this.api.ytHistory(200));
const items = (res?.items || []).map((v: any) => ({
videoId: String(v.videoId),
title: String(v.title || v.videoId).slice(0, 300),
watchedAt: String(v.watchedAt || new Date().toISOString()),
}));
if (!items.length) {
this.ytAutoError.set('Aucune entrée renvoyée par YouTube (historique vide ou suspendu côté Google ?).');
return;
}
this.takeoutItems.set(items);
this.takeoutFileName.set('sync auto YouTube');
await this.importTakeout();
} catch (e: any) {
const code = e?.error?.error || e?.message || 'ythistory_fetch_failed';
const hint = e?.error?.hint;
this.ytAutoError.set(hint ? `${code} — ${hint}` : code);
} finally {
this.ytAutoLoading.set(false);
}
}
takeoutRange(): { from: string; to: string } | null {
const items = this.takeoutItems();
@@ -213,7 +246,9 @@ export class ImportComponent implements OnInit {
this.wlLoading.set(false);
},
error: (e) => {
this.wlError.set(e?.error?.error || e?.message || 'watchlater_fetch_failed');
const code = e?.error?.error || e?.message || 'watchlater_fetch_failed';
const hint = e?.error?.hint;
this.wlError.set(hint ? `${code} — ${hint}` : code);
this.wlLoading.set(false);
},
});
+8
View File
@@ -19,6 +19,9 @@ export interface OAuthPreview {
likes: Array<{ provider: string; videoId: string; title: string; thumbnail?: string }>;
subscriptionCount: number;
likeCount: number;
/** Totaux du compte (l'aperçu n'affiche que les premiers, l'import prend tout). */
subscriptionTotal?: number;
likeTotal?: number;
}
export interface OAuthImportResult {
@@ -74,6 +77,11 @@ export class OAuthImportService {
importTakeoutBatch(items: Array<{ videoId: string; title?: string; watchedAt?: string }>): Observable<{ imported: number; skipped: number; total: number }> {
return this.http.post<any>(`/api/user/history/takeout`, { items });
}
/** Historique YouTube via InnerTube authentifié (sync auto, sans fichier). */
ytHistory(max = 200): Observable<{ items: Array<{ videoId: string; title: string; thumbnail?: string; watchedAt?: string }>; count: number; hasMore: boolean }> {
return this.http.get<any>(`/api/oauth/google/yt-history?max=${max}`);
}
}
/** Parse local du `watch-history.json` de Google Takeout (confidentiel : rien n'est uploadé tel quel). */