From 050cea3a92f8ff69204c8141d3058ef61a1f4919 Mon Sep 17 00:00:00 2001 From: Bruno Charest Date: Sat, 26 Sep 2026 22:25:50 -0400 Subject: [PATCH] =?UTF-8?q?fix(sync):=20WL=20diagnostic,=20import-all=20pa?= =?UTF-8?q?gin=C3=A9=20+=20totaux,=20sync=20auto=20historique=20InnerTube?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Watch Later: youtube_no_channel / watchlater_not_found avec hint actionnable + log cles relatedPlaylists (sans PII) - Preview Google: totaux compte (subscriptionTotal/likeTotal), UI 'X / N, l'import prend tout'; import-all paginé (subs<=1000, likes<=500) au lieu de 50/25 - Sync auto historique: GET /oauth/google/yt-history (browse FEhistory authentifie, continuations, <=500), bouton Sync auto dans carte Takeout, import via lots takeout existants --- server/index.mjs | 67 ++++++- server/oauth.mjs | 167 +++++++++++++++--- .../library/import/import.component.html | 14 +- .../library/import/import.component.ts | 37 +++- src/services/oauth-import.service.ts | 8 + 5 files changed, 260 insertions(+), 33 deletions(-) diff --git a/server/index.mjs b/server/index.mjs index 4a25ee3..f5af064 100644 --- a/server/index.mjs +++ b/server/index.mjs @@ -105,7 +105,7 @@ import { oauthStatus, buildAuthUrl, createOAuthState, consumeOAuthState, exchangeCode, refreshAccessToken, redirectUriFor, fetchGoogleProfile, fetchGoogleSubscriptions, fetchGoogleLiked, fetchTwitchProfile, fetchTwitchFollows, hasGoogleWriteScope, - fetchGoogleWatchLater, pushGoogleWatchLater, + fetchGoogleWatchLater, pushGoogleWatchLater, fetchInnerTubeHistory, } from './oauth.mjs'; import { getSearchMode, getYtDlpBin, hasCookiesFile, metricsSnapshot } from './providers/youtube-common.mjs'; import { ytScrapeCacheStats } from './providers/youtube.mjs'; @@ -564,11 +564,20 @@ r.get('/oauth/:provider/preview', authMiddlewareCookieAware, oauthLimiter, async const provider = requireOAuthProvider(req.params.provider); const conn = await freshOAuthToken(req.user.id, provider); if (provider === 'google') { - const [subscriptions, likes] = await Promise.all([ - fetchGoogleSubscriptions(conn.accessToken, 50).catch(() => []), - fetchGoogleLiked(conn.accessToken, 25).catch(() => []), + const [subsRes, likesRes] = await Promise.all([ + fetchGoogleSubscriptions(conn.accessToken, 50).catch(() => ({ items: [], total: 0 })), + fetchGoogleLiked(conn.accessToken, 25).catch(() => ({ items: [], total: 0 })), ]); - return res.json({ provider, subscriptions, likes, subscriptionCount: subscriptions.length, likeCount: likes.length }); + return res.json({ + provider, + subscriptions: subsRes.items, + likes: likesRes.items, + subscriptionCount: subsRes.items.length, + likeCount: likesRes.items.length, + // Totaux du compte (l'aperçu n'affiche que les 50/25 premiers, l'import prend tout). + subscriptionTotal: subsRes.total, + likeTotal: likesRes.total, + }); } const profile = await fetchTwitchProfile(conn.accessToken).catch(() => ({ id: conn.externalUserId || '' })); const follows = await fetchTwitchFollows(conn.accessToken, profile?.id || conn.externalUserId || '', 100); @@ -594,7 +603,8 @@ r.post('/oauth/:provider/import', authMiddlewareCookieAware, oauthLimiter, async if (provider === 'google') { if (wantSubs) { - const subs = await fetchGoogleSubscriptions(conn.accessToken, 50); + // Import complet paginé (l'aperçu n'en montre que 50). + const { items: subs } = await fetchGoogleSubscriptions(conn.accessToken, 1000); for (const s of subs) { try { const row = upsertChannelRow({ @@ -608,7 +618,8 @@ r.post('/oauth/:provider/import', authMiddlewareCookieAware, oauthLimiter, async } } if (wantLikes) { - const likes = await fetchGoogleLiked(conn.accessToken, 25); + // Import complet paginé (l'aperçu n'en montre que 25). + const { items: likes } = await fetchGoogleLiked(conn.accessToken, 500); for (const v of likes) { try { likeVideo({ userId: req.user.id, provider: 'youtube', videoId: v.videoId, title: v.title, thumbnail: v.thumbnail }); @@ -655,7 +666,47 @@ r.get('/oauth/google/watchlater', authMiddlewareCookieAware, oauthLimiter, async writeGranted: hasGoogleWriteScope(conn.scopes), }); } catch (error) { - return res.status(error?.status || 502).json({ error: error?.message || 'watchlater_fetch_failed' }); + const out = { error: error?.message || 'watchlater_fetch_failed' }; + if (error?.hint) out.hint = error.hint; + return res.status(error?.status || 502).json(out); + } +}); + +// -------------------- Google : historique auto via InnerTube authentifié -------------------- +// L'API Data v3 n'expose pas l'historique : on lit youtubei/v1/browse +// (FEhistory) avec le Bearer OAuth de l'utilisateur (mécanisme SmartTube). +// Le client réutilise ensuite POST /user/history/takeout pour l'importer. + +function innertubeApiKey() { + const single = String(process.env.YOUTUBE_API_KEY || '').trim(); + if (single && !single.includes(',')) return single; + const csv = String(process.env.YOUTUBE_API_KEYS || '').trim(); + try { + if (csv.startsWith('[')) { + const arr = JSON.parse(csv); + if (Array.isArray(arr) && arr[0]) return String(arr[0]); + } + } catch {} + if (csv) { + const first = csv.split(',').map((s) => s.trim()).filter(Boolean)[0]; + if (first) return first; + } + if (single) return single.split(',').map((s) => s.trim()).filter(Boolean)[0] || ''; + return ''; +} + +r.get('/oauth/google/yt-history', authMiddlewareCookieAware, oauthLimiter, async (req, res) => { + try { + const conn = await freshOAuthToken(req.user.id, 'google'); + const max = Math.min(500, Math.max(1, Number(req.query.max || 200))); + const { items, hasMore } = await fetchInnerTubeHistory(conn.accessToken, innertubeApiKey(), max); + return res.json({ items, count: items.length, hasMore }); + } catch (error) { + const out = { error: error?.message || 'ythistory_fetch_failed' }; + if (error?.status === 401) { + out.hint = 'Session Google expirée : déconnectez puis reconnectez Google.'; + } + return res.status(error?.status || 502).json(out); } }); diff --git a/server/oauth.mjs b/server/oauth.mjs index a86ab74..9108eeb 100644 --- a/server/oauth.mjs +++ b/server/oauth.mjs @@ -254,8 +254,9 @@ export async function fetchGoogleProfile(accessToken) { }; } -export async function fetchGoogleSubscriptions(accessToken, max = 50) { +export async function fetchGoogleSubscriptions(accessToken, max = 1000) { const out = []; + let total = 0; let pageToken = ''; while (out.length < max) { const qs = new URLSearchParams({ @@ -266,6 +267,7 @@ export async function fetchGoogleSubscriptions(accessToken, max = 50) { }); if (pageToken) qs.set('pageToken', pageToken); const data = await getJson(`https://www.googleapis.com/youtube/v3/subscriptions?${qs.toString()}`, accessToken); + if (!total && typeof data?.pageInfo?.totalResults === 'number') total = data.pageInfo.totalResults; for (const item of Array.isArray(data?.items) ? data.items : []) { const sn = item?.snippet || {}; const channelId = sn?.resourceId?.channelId || ''; @@ -282,29 +284,37 @@ export async function fetchGoogleSubscriptions(accessToken, max = 50) { pageToken = data?.nextPageToken || ''; if (!pageToken) break; } - return out; + return { items: out, total: total || out.length }; } -export async function fetchGoogleLiked(accessToken, max = 25) { - const qs = new URLSearchParams({ - part: 'snippet,contentDetails', - myRating: 'like', - maxResults: String(Math.min(50, max)), - }); - const data = await getJson(`https://www.googleapis.com/youtube/v3/videos?${qs.toString()}`, accessToken); +export async function fetchGoogleLiked(accessToken, max = 500) { const out = []; - for (const item of Array.isArray(data?.items) ? data.items : []) { - const id = item?.id || ''; - const sn = item?.snippet || {}; - if (!id) continue; - out.push({ - provider: 'youtube', - videoId: String(id), - title: sn?.title || '', - thumbnail: sn?.thumbnails?.medium?.url || sn?.thumbnails?.default?.url || '', + let total = 0; + let pageToken = ''; + while (out.length < max) { + const qs = new URLSearchParams({ + part: 'snippet,contentDetails', + myRating: 'like', + maxResults: String(Math.min(50, max - out.length)), }); + if (pageToken) qs.set('pageToken', pageToken); + const data = await getJson(`https://www.googleapis.com/youtube/v3/videos?${qs.toString()}`, accessToken); + if (!total && typeof data?.pageInfo?.totalResults === 'number') total = data.pageInfo.totalResults; + for (const item of Array.isArray(data?.items) ? data.items : []) { + const id = item?.id || ''; + const sn = item?.snippet || {}; + if (!id) continue; + out.push({ + provider: 'youtube', + videoId: String(id), + title: sn?.title || '', + thumbnail: sn?.thumbnails?.medium?.url || sn?.thumbnails?.default?.url || '', + }); + } + pageToken = data?.nextPageToken || ''; + if (!pageToken) break; } - return out; + return { items: out, total: total || out.length }; } // -------------------- Google : Watch Later -------------------- @@ -337,8 +347,23 @@ async function googleApiPost(accessToken, path, body) { export async function fetchGoogleWatchLaterId(accessToken) { const data = await googleApiGet(accessToken, 'channels', { part: 'contentDetails', mine: 'true' }); const item = Array.isArray(data?.items) ? data.items[0] : null; + if (!item) { + throw Object.assign(new Error('youtube_no_channel'), { + status: 502, + hint: 'Ce compte Google n’a pas de chaîne YouTube : créez-en une sur youtube.com pour utiliser Regarder plus tard.', + }); + } const id = item?.contentDetails?.relatedPlaylists?.watchLater || ''; - if (!id) throw Object.assign(new Error('watchlater_not_found'), { status: 502 }); + if (!id) { + // Diagnostic sans PII : quelles playlists liées YouTube expose-t-il ? + try { + console.warn('[oauth] watchLater absent, relatedPlaylists =', Object.keys(item?.contentDetails?.relatedPlaylists || {}).join(',') || '(vide)'); + } catch {} + throw Object.assign(new Error('watchlater_not_found'), { + status: 502, + hint: 'YouTube ne renvoie pas de playlist « Regarder plus tard » pour ce compte (compte de marque sans chaîne principale ?). Les abonnements et favoris restent importables.', + }); + } return String(id); } @@ -384,6 +409,108 @@ export async function pushGoogleWatchLater(accessToken, videoId) { return { playlistItemId: data?.id || null }; } +// -------------------- Historique YouTube via InnerTube authentifié -------------------- +// L'API Data v3 n'expose pas l'historique : on passe par youtubei/v1/browse +// (browseId FEhistory) avec le Bearer OAuth de l'utilisateur — même mécanisme +// que SmartTube. Lecture seule, pas de scope supplémentaire. + +function pickThumb(thumbnails) { + const list = Array.isArray(thumbnails) ? thumbnails : []; + let best = null; + for (const t of list) { + if (!t?.url) continue; + if (!best || Number(t.width || 0) > Number(best.width || 0)) best = t; + } + return best?.url || ''; +} + +function runsText(runs) { + try { + return (Array.isArray(runs) ? runs : []).map((r) => r?.text || '').join(''); + } catch { return ''; } +} + +function extractHistoryEntries(node, out) { + if (!node || typeof node !== 'object') return; + if (Array.isArray(node)) { + for (const e of node) extractHistoryEntries(e, out); + return; + } + if (node.videoRenderer?.videoId) { + const vr = node.videoRenderer; + out.push({ + provider: 'youtube', + videoId: String(vr.videoId), + title: runsText(vr.title?.runs) || String(vr.title?.simpleText || vr.videoId), + thumbnail: pickThumb(vr.thumbnail?.thumbnails), + watchedAt: new Date().toISOString(), + }); + return; + } + for (const v of Object.values(node)) { + if (v && typeof v === 'object') extractHistoryEntries(v, out); + } +} + +function findHistoryContinuation(data) { + try { + const tabs = data?.contents?.singleColumnBrowseResultsRenderer?.tabs || []; + const stack = [...tabs]; + while (stack.length) { + const n = stack.pop(); + if (!n || typeof n !== 'object') continue; + if (Array.isArray(n)) { stack.push(...n); continue; } + const token = n?.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token; + if (token) return String(token); + for (const v of Object.values(n)) { + if (v && typeof v === 'object') stack.push(v); + } + } + } catch {} + return null; +} + +export async function fetchInnerTubeHistory(accessToken, apiKey, max = 200) { + const key = String(apiKey || '').trim(); + if (!key) throw Object.assign(new Error('youtube_api_key_unavailable'), { status: 503 }); + const out = []; + const seen = new Set(); + let continuation = null; + let pages = 0; + while (out.length < max && pages < 10) { + pages++; + const body = continuation + ? { context: { client: { clientName: 'WEB', clientVersion: '2.20260101.00.00', hl: 'fr', gl: 'FR' } }, continuation } + : { context: { client: { clientName: 'WEB', clientVersion: '2.20260101.00.00', hl: 'fr', gl: 'FR' } }, browseId: 'FEhistory' }; + const resp = await fetch(`https://www.youtube.com/youtubei/v1/browse?key=${encodeURIComponent(key)}&prettyPrint=false`, { + method: 'POST', + headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(20000), + }); + const data = await resp.json().catch(() => ({})); + if (!resp.ok) { + const err = new Error(`innertube_history_failed_${resp.status}`); + err.status = resp.status === 401 ? 401 : 502; + throw err; + } + const batch = []; + const root = continuation + ? (data?.onResponseReceivedActions || data?.continuationContents) + : data; + extractHistoryEntries(root, batch); + for (const v of batch) { + if (seen.has(v.videoId)) continue; + seen.add(v.videoId); + out.push(v); + if (out.length >= max) break; + } + continuation = findHistoryContinuation(data); + if (!continuation) break; + } + return { items: out.slice(0, max), hasMore: Boolean(continuation) }; +} + // -------------------- Twitch -------------------- function twitchClientId() { diff --git a/src/components/library/import/import.component.html b/src/components/library/import/import.component.html index 4bc8692..3f21214 100644 --- a/src/components/library/import/import.component.html +++ b/src/components/library/import/import.component.html @@ -69,7 +69,8 @@
{{ errors()[p.id] }}
-

Abonnements : {{ pv.subscriptionCount }} • Favoris : {{ pv.likeCount }}

+

Abonnements : {{ pv.subscriptionCount }} / {{ pv.subscriptionTotal }} • Favoris : {{ pv.likeCount }} / {{ pv.likeTotal }}

+

Aperçu limité — le bouton Importer prend tout le compte.