Projets/NewTube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(sync): WL diagnostic, import-all paginé + totaux, sync auto historique InnerTube

Browse Source 
- Watch Later: youtube_no_channel / watchlater_not_found avec hint
  actionnable + log cles relatedPlaylists (sans PII)
- Preview Google: totaux compte (subscriptionTotal/likeTotal), UI
  'X / N, l'import prend tout'; import-all paginé (subs<=1000,
  likes<=500) au lieu de 50/25
- Sync auto historique: GET /oauth/google/yt-history (browse FEhistory
  authentifie, continuations, <=500), bouton Sync auto dans carte
  Takeout, import via lots takeout existants
This commit is contained in:
bruno
2026-09-26 22:25:50 -04:00
parent 1c013452fa
commit 050cea3a92
5 changed files with 260 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/index.mjs
+59 -8
View File
@@ -105,7 +105,7 @@ import {
oauthStatus, buildAuthUrl, createOAuthState, consumeOAuthState, exchangeCode,
refreshAccessToken, redirectUriFor, fetchGoogleProfile, fetchGoogleSubscriptions,
fetchGoogleLiked, fetchTwitchProfile, fetchTwitchFollows, hasGoogleWriteScope,
fetchGoogleWatchLater, pushGoogleWatchLater,
fetchGoogleWatchLater, pushGoogleWatchLater, fetchInnerTubeHistory,
} from './oauth.mjs';
import { getSearchMode, getYtDlpBin, hasCookiesFile, metricsSnapshot } from './providers/youtube-common.mjs';
import { ytScrapeCacheStats } from './providers/youtube.mjs';
@@ -564,11 +564,20 @@ r.get('/oauth/:provider/preview', authMiddlewareCookieAware, oauthLimiter, async
const provider = requireOAuthProvider(req.params.provider);
const conn = await freshOAuthToken(req.user.id, provider);
if (provider === 'google') {
const [subscriptions, likes] = await Promise.all([
fetchGoogleSubscriptions(conn.accessToken, 50).catch(() => []),
fetchGoogleLiked(conn.accessToken, 25).catch(() => []),
const [subsRes, likesRes] = await Promise.all([
fetchGoogleSubscriptions(conn.accessToken, 50).catch(() => ({ items: [], total: 0 })),
fetchGoogleLiked(conn.accessToken, 25).catch(() => ({ items: [], total: 0 })),
]);
return res.json({ provider, subscriptions, likes, subscriptionCount: subscriptions.length, likeCount: likes.length });
return res.json({
provider,
subscriptions: subsRes.items,
likes: likesRes.items,
subscriptionCount: subsRes.items.length,
likeCount: likesRes.items.length,
// Totaux du compte (l'aperçu n'affiche que les 50/25 premiers, l'import prend tout).
subscriptionTotal: subsRes.total,
likeTotal: likesRes.total,
});
}
const profile = await fetchTwitchProfile(conn.accessToken).catch(() => ({ id: conn.externalUserId || '' }));
const follows = await fetchTwitchFollows(conn.accessToken, profile?.id || conn.externalUserId || '', 100);
@@ -594,7 +603,8 @@ r.post('/oauth/:provider/import', authMiddlewareCookieAware, oauthLimiter, async
if (provider === 'google') {
if (wantSubs) {
const subs = await fetchGoogleSubscriptions(conn.accessToken, 50);
// Import complet paginé (l'aperçu n'en montre que 50).
const { items: subs } = await fetchGoogleSubscriptions(conn.accessToken, 1000);
for (const s of subs) {
try {
const row = upsertChannelRow({
@@ -608,7 +618,8 @@ r.post('/oauth/:provider/import', authMiddlewareCookieAware, oauthLimiter, async
}
}
if (wantLikes) {
const likes = await fetchGoogleLiked(conn.accessToken, 25);
// Import complet paginé (l'aperçu n'en montre que 25).
const { items: likes } = await fetchGoogleLiked(conn.accessToken, 500);
for (const v of likes) {
try {
likeVideo({ userId: req.user.id, provider: 'youtube', videoId: v.videoId, title: v.title, thumbnail: v.thumbnail });
@@ -655,7 +666,47 @@ r.get('/oauth/google/watchlater', authMiddlewareCookieAware, oauthLimiter, async
writeGranted: hasGoogleWriteScope(conn.scopes),
});
} catch (error) {
return res.status(error?.status || 502).json({ error: error?.message || 'watchlater_fetch_failed' });
const out = { error: error?.message || 'watchlater_fetch_failed' };
if (error?.hint) out.hint = error.hint;
return res.status(error?.status || 502).json(out);
}
});
// -------------------- Google : historique auto via InnerTube authentifié --------------------
// L'API Data v3 n'expose pas l'historique : on lit youtubei/v1/browse
// (FEhistory) avec le Bearer OAuth de l'utilisateur (mécanisme SmartTube).
// Le client réutilise ensuite POST /user/history/takeout pour l'importer.
function innertubeApiKey() {
const single = String(process.env.YOUTUBE_API_KEY || '').trim();
if (single && !single.includes(',')) return single;
const csv = String(process.env.YOUTUBE_API_KEYS || '').trim();
try {
if (csv.startsWith('[')) {
const arr = JSON.parse(csv);
if (Array.isArray(arr) && arr[0]) return String(arr[0]);
}
} catch {}
if (csv) {
const first = csv.split(',').map((s) => s.trim()).filter(Boolean)[0];
if (first) return first;
}
if (single) return single.split(',').map((s) => s.trim()).filter(Boolean)[0] || '';
return '';
}
r.get('/oauth/google/yt-history', authMiddlewareCookieAware, oauthLimiter, async (req, res) => {
try {
const conn = await freshOAuthToken(req.user.id, 'google');
const max = Math.min(500, Math.max(1, Number(req.query.max || 200)));
const { items, hasMore } = await fetchInnerTubeHistory(conn.accessToken, innertubeApiKey(), max);
return res.json({ items, count: items.length, hasMore });
} catch (error) {
const out = { error: error?.message || 'ythistory_fetch_failed' };
if (error?.status === 401) {
out.hint = 'Session Google expirée : déconnectez puis reconnectez Google.';
}
return res.status(error?.status || 502).json(out);
}
});
server/oauth.mjs
+147 -20
View File
@@ -254,8 +254,9 @@ export async function fetchGoogleProfile(accessToken) {
};
}
export async function fetchGoogleSubscriptions(accessToken, max = 50) {
export async function fetchGoogleSubscriptions(accessToken, max = 1000) {
const out = [];
let total = 0;
let pageToken = '';
while (out.length < max) {
const qs = new URLSearchParams({
@@ -266,6 +267,7 @@ export async function fetchGoogleSubscriptions(accessToken, max = 50) {
});
if (pageToken) qs.set('pageToken', pageToken);
const data = await getJson(`https://www.googleapis.com/youtube/v3/subscriptions?${qs.toString()}`, accessToken);
if (!total && typeof data?.pageInfo?.totalResults === 'number') total = data.pageInfo.totalResults;
for (const item of Array.isArray(data?.items) ? data.items : []) {
const sn = item?.snippet || {};
const channelId = sn?.resourceId?.channelId || '';
@@ -282,29 +284,37 @@ export async function fetchGoogleSubscriptions(accessToken, max = 50) {
pageToken = data?.nextPageToken || '';
if (!pageToken) break;
}
return out;
return { items: out, total: total || out.length };
}
export async function fetchGoogleLiked(accessToken, max = 25) {
const qs = new URLSearchParams({
part: 'snippet,contentDetails',
myRating: 'like',
maxResults: String(Math.min(50, max)),
});
const data = await getJson(`https://www.googleapis.com/youtube/v3/videos?${qs.toString()}`, accessToken);
export async function fetchGoogleLiked(accessToken, max = 500) {
const out = [];
for (const item of Array.isArray(data?.items) ? data.items : []) {
const id = item?.id || '';
const sn = item?.snippet || {};
if (!id) continue;
out.push({
provider: 'youtube',
videoId: String(id),
title: sn?.title || '',
thumbnail: sn?.thumbnails?.medium?.url || sn?.thumbnails?.default?.url || '',
let total = 0;
let pageToken = '';
while (out.length < max) {
const qs = new URLSearchParams({
part: 'snippet,contentDetails',
myRating: 'like',
maxResults: String(Math.min(50, max - out.length)),
});
if (pageToken) qs.set('pageToken', pageToken);
const data = await getJson(`https://www.googleapis.com/youtube/v3/videos?${qs.toString()}`, accessToken);
if (!total && typeof data?.pageInfo?.totalResults === 'number') total = data.pageInfo.totalResults;
for (const item of Array.isArray(data?.items) ? data.items : []) {
const id = item?.id || '';
const sn = item?.snippet || {};
if (!id) continue;
out.push({
provider: 'youtube',
videoId: String(id),
title: sn?.title || '',
thumbnail: sn?.thumbnails?.medium?.url || sn?.thumbnails?.default?.url || '',
});
}
pageToken = data?.nextPageToken || '';
if (!pageToken) break;
}
return out;
return { items: out, total: total || out.length };
}
// -------------------- Google : Watch Later --------------------
@@ -337,8 +347,23 @@ async function googleApiPost(accessToken, path, body) {
export async function fetchGoogleWatchLaterId(accessToken) {
const data = await googleApiGet(accessToken, 'channels', { part: 'contentDetails', mine: 'true' });
const item = Array.isArray(data?.items) ? data.items[0] : null;
if (!item) {
throw Object.assign(new Error('youtube_no_channel'), {
status: 502,
hint: 'Ce compte Google n’a pas de chaîne YouTube : créez-en une sur youtube.com pour utiliser Regarder plus tard.',
});
}
const id = item?.contentDetails?.relatedPlaylists?.watchLater || '';
if (!id) throw Object.assign(new Error('watchlater_not_found'), { status: 502 });
if (!id) {
// Diagnostic sans PII : quelles playlists liées YouTube expose-t-il ?
try {
console.warn('[oauth] watchLater absent, relatedPlaylists =', Object.keys(item?.contentDetails?.relatedPlaylists || {}).join(',') || '(vide)');
} catch {}
throw Object.assign(new Error('watchlater_not_found'), {
status: 502,
hint: 'YouTube ne renvoie pas de playlist « Regarder plus tard » pour ce compte (compte de marque sans chaîne principale ?). Les abonnements et favoris restent importables.',
});
}
return String(id);
}
@@ -384,6 +409,108 @@ export async function pushGoogleWatchLater(accessToken, videoId) {
return { playlistItemId: data?.id || null };
}
// -------------------- Historique YouTube via InnerTube authentifié --------------------
// L'API Data v3 n'expose pas l'historique : on passe par youtubei/v1/browse
// (browseId FEhistory) avec le Bearer OAuth de l'utilisateur — même mécanisme
// que SmartTube. Lecture seule, pas de scope supplémentaire.
function pickThumb(thumbnails) {
const list = Array.isArray(thumbnails) ? thumbnails : [];
let best = null;
for (const t of list) {
if (!t?.url) continue;
if (!best || Number(t.width || 0) > Number(best.width || 0)) best = t;
}
return best?.url || '';
}
function runsText(runs) {
try {
return (Array.isArray(runs) ? runs : []).map((r) => r?.text || '').join('');
} catch { return ''; }
}
function extractHistoryEntries(node, out) {
if (!node || typeof node !== 'object') return;
if (Array.isArray(node)) {
for (const e of node) extractHistoryEntries(e, out);
return;
}
if (node.videoRenderer?.videoId) {
const vr = node.videoRenderer;
out.push({
provider: 'youtube',
videoId: String(vr.videoId),
title: runsText(vr.title?.runs) || String(vr.title?.simpleText || vr.videoId),
thumbnail: pickThumb(vr.thumbnail?.thumbnails),
watchedAt: new Date().toISOString(),
});
return;
}
for (const v of Object.values(node)) {
if (v && typeof v === 'object') extractHistoryEntries(v, out);
}
}
function findHistoryContinuation(data) {
try {
const tabs = data?.contents?.singleColumnBrowseResultsRenderer?.tabs || [];
const stack = [...tabs];
while (stack.length) {
const n = stack.pop();
if (!n || typeof n !== 'object') continue;
if (Array.isArray(n)) { stack.push(...n); continue; }
const token = n?.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token;
if (token) return String(token);
for (const v of Object.values(n)) {
if (v && typeof v === 'object') stack.push(v);
}
}
} catch {}
return null;
}
export async function fetchInnerTubeHistory(accessToken, apiKey, max = 200) {
const key = String(apiKey || '').trim();
if (!key) throw Object.assign(new Error('youtube_api_key_unavailable'), { status: 503 });
const out = [];
const seen = new Set();
let continuation = null;
let pages = 0;
while (out.length < max && pages < 10) {
pages++;
const body = continuation
? { context: { client: { clientName: 'WEB', clientVersion: '2.20260101.00.00', hl: 'fr', gl: 'FR' } }, continuation }
: { context: { client: { clientName: 'WEB', clientVersion: '2.20260101.00.00', hl: 'fr', gl: 'FR' } }, browseId: 'FEhistory' };
const resp = await fetch(`https://www.youtube.com/youtubei/v1/browse?key=${encodeURIComponent(key)}&prettyPrint=false`, {
method: 'POST',
headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json' },
body: JSON.stringify(body),
signal: AbortSignal.timeout(20000),
});
const data = await resp.json().catch(() => ({}));
if (!resp.ok) {
const err = new Error(`innertube_history_failed_${resp.status}`);
err.status = resp.status === 401 ? 401 : 502;
throw err;
}
const batch = [];
const root = continuation
? (data?.onResponseReceivedActions || data?.continuationContents)
: data;
extractHistoryEntries(root, batch);
for (const v of batch) {
if (seen.has(v.videoId)) continue;
seen.add(v.videoId);
out.push(v);
if (out.length >= max) break;
}
continuation = findHistoryContinuation(data);
if (!continuation) break;
}
return { items: out.slice(0, max), hasMore: Boolean(continuation) };
}
// -------------------- Twitch --------------------
function twitchClientId() {