feat(channel): page details chaine multi-providers avec onglets et infinite scroll
CI / build-and-test (push) Successful in 13m34s
CI / build-and-test (push) Successful in 13m34s
- Strategy Pattern front: ChannelProviderInterface + 6 providers (yt/dm/tw/pt/od/ru) + factory (ids longs/courts) - ChannelContentService: cache par onglet/tri/recherche (TTL 5min), pagination nextPage - ChannelPage: header (banniere/avatar/subs/description ...plus), tabs dynamiques par capabilities, grilles videos/shorts 9:16/playlists/live, recherche debounced, tri recent/populaire, infinite scroll IntersectionObserver, skeletons/empty/error - Backend: GET /api/channels/:provider/:id/content (videos/shorts/playlists/live) natif par provider
This commit is contained in:
+106
-18
@@ -2,6 +2,10 @@ import express from 'express';
|
||||
import * as cheerio from 'cheerio';
|
||||
import axios from 'axios';
|
||||
import rateLimit from 'express-rate-limit';
|
||||
import { spawn } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
@@ -33,22 +37,87 @@ function getCache(key) {
|
||||
}
|
||||
|
||||
/* ------------------------------- HTTP GET -------------------------------- */
|
||||
/**
|
||||
* Stratégie best-effort (cf. server/providers/rumble.mjs) :
|
||||
* 1. axios/Node avec headers navigateur (rapide quand CF ne challenge pas) ;
|
||||
* 2. helper python3 + curl_cffi (impersonation Chrome) — indispensable derrière
|
||||
* Cloudflare (les pages /search/* retournent sinon 403 "Just a moment").
|
||||
* Retourne le HTML brut ou lève une erreur si tout est challengé.
|
||||
*/
|
||||
const ROUTER_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROUTER_PY_HELPER = path.join(ROUTER_DIR, 'providers', 'rumble_fetch.py');
|
||||
const ROUTER_PY_HELPER_EXISTS = fs.existsSync(ROUTER_DIR) && fs.existsSync(ROUTER_PY_HELPER);
|
||||
|
||||
function isChallengeHtml(html) {
|
||||
return /Just a moment|challenge-platform|cf-chl/i.test(String(html || '').slice(0, 4000));
|
||||
}
|
||||
|
||||
async function pythonFetchHtml(url, { timeoutMs = 20_000 } = {}) {
|
||||
if (!ROUTER_PY_HELPER_EXISTS) return null;
|
||||
for (const py of ['python3', 'python']) {
|
||||
const result = await new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const done = (v) => { if (!settled) { settled = true; resolve(v); } };
|
||||
let child;
|
||||
try {
|
||||
child = spawn(py, [ROUTER_PY_HELPER, url, String(Math.ceil(timeoutMs / 1000))], {
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
});
|
||||
} catch { return done(null); }
|
||||
let stdout = '';
|
||||
let failed = false;
|
||||
const timer = setTimeout(() => { failed = true; try { child.kill(); } catch {} done(null); }, timeoutMs + 5_000);
|
||||
child.stdout.on('data', (d) => { stdout += d.toString(); });
|
||||
child.on('error', () => { clearTimeout(timer); done(null); });
|
||||
child.on('close', (code) => {
|
||||
clearTimeout(timer);
|
||||
if (failed) return;
|
||||
done(code === 0 && stdout.length > 1000 ? stdout : null);
|
||||
});
|
||||
});
|
||||
if (result) return result;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function httpGet(url) {
|
||||
const resp = await axios.get(url, {
|
||||
headers: {
|
||||
// UA “desktop” moderne pour minimiser les anti-bot simples
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36',
|
||||
'Accept':
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.8'
|
||||
},
|
||||
timeout: 15000,
|
||||
// Important: pas de redirects inter-domain hasardeux
|
||||
maxRedirects: 3,
|
||||
validateStatus: s => s >= 200 && s < 400
|
||||
});
|
||||
return resp.data;
|
||||
// Tentative 1 : axios/Node direct
|
||||
try {
|
||||
const resp = await axios.get(url, {
|
||||
headers: {
|
||||
// UA “desktop” moderne pour minimiser les anti-bot simples
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
'Accept':
|
||||
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'sec-ch-ua': '"Chromium";v="131", "Not_A Brand";v="24"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"Windows"',
|
||||
'sec-fetch-dest': 'document',
|
||||
'sec-fetch-mode': 'navigate',
|
||||
'sec-fetch-site': 'none',
|
||||
'sec-fetch-user': '?1',
|
||||
'upgrade-insecure-requests': '1',
|
||||
},
|
||||
timeout: 15000,
|
||||
// Important: pas de redirects inter-domain hasardeux
|
||||
maxRedirects: 3,
|
||||
validateStatus: s => s >= 200 && s < 400
|
||||
});
|
||||
const html = resp.data;
|
||||
if (typeof html === 'string' && html.length > 1000 && !isChallengeHtml(html)) return html;
|
||||
} catch (e) {
|
||||
// Statuts 403 Cloudflare → on bascule vers le helper python ci-dessous
|
||||
const status = e?.response?.status;
|
||||
if (status && status !== 403) throw e;
|
||||
}
|
||||
// Tentative 2 : python + curl_cffi (impersonation navigateur)
|
||||
const pyHtml = await pythonFetchHtml(url);
|
||||
if (pyHtml) return pyHtml;
|
||||
const err = new Error('Request failed with status code 403');
|
||||
err.status = 403;
|
||||
throw err;
|
||||
}
|
||||
|
||||
/* ------------------------- Utils: normalisation ID ------------------------ */
|
||||
@@ -223,6 +292,8 @@ async function scrapeRumbleVideo(videoIdOrUrl) {
|
||||
/* ------------------ Scraper de liste (search / browse) ------------------ */
|
||||
function parseDurationToSeconds(text) {
|
||||
if (!text) return 0;
|
||||
// Garde-fou : les attributs datetime (dates de publication) ne sont jamais des durées.
|
||||
if (/^\d{4}-\d{2}-\d{2}/.test(String(text).trim())) return 0;
|
||||
|
||||
// Nettoyer le texte en supprimant les espaces et caractères non numériques inutiles
|
||||
const cleanText = text.trim().replace(/\s+/g, '');
|
||||
@@ -299,7 +370,19 @@ async function scrapeRumbleList({ q, page = 1, limit = 24, sort = 'viral' }) {
|
||||
|
||||
const card = $(el).closest('li, article, .video-listing-entry, .video-item, .video-card, div');
|
||||
|
||||
const title = (($(el).attr('title') || '') + ' ' + $(el).text()).trim() || card.find('h3, h2, .video-item--title').first().text().trim();
|
||||
// Titre : préférer le vrai titre de la carte (h3/h2) au texte brut du lien
|
||||
// (le lien contient souvent des badges "N watching" / durées qui polluaient le titre).
|
||||
const cardTitle = card.find('h3, h2, .video-item--title').first().text().replace(/\s+/g, ' ').trim();
|
||||
const linkTitle = ($(el).attr('title') || '').replace(/\s+/g, ' ').trim();
|
||||
const linkText = $(el).text().replace(/\s+/g, ' ').trim();
|
||||
let title = cardTitle || linkTitle || linkText;
|
||||
// Filtrer les faux titres (badges viewers, durées seules, chaînes vides)
|
||||
if (!title || /^\d+\s+watching$/i.test(title) || /^[0-9:.,\s]+$/.test(title) || title.length < 3) {
|
||||
// Dernier recours : attribut alt de l'image de la carte
|
||||
const alt = (card.find('img').first().attr('alt') || '').replace(/\s+/g, ' ').trim();
|
||||
if (alt && alt.length >= 3 && !/^\d+\s+watching$/i.test(alt)) title = alt;
|
||||
else return; // ignorer cette carte (badge live, doublon de lien, …)
|
||||
}
|
||||
|
||||
// Thumb robuste: data-src > src
|
||||
let thumb =
|
||||
@@ -320,9 +403,9 @@ async function scrapeRumbleList({ q, page = 1, limit = 24, sort = 'viral' }) {
|
||||
const durationCandidates = [];
|
||||
if (durationElement.length) {
|
||||
durationCandidates.push(
|
||||
durationElement.attr('data-value'), // motif Rumble : <span class="video-item--duration" data-value="3:06:42">
|
||||
durationElement.attr('data-duration'),
|
||||
durationElement.attr('data-time'),
|
||||
durationElement.attr('datetime'),
|
||||
durationElement.attr('aria-label'),
|
||||
durationElement.attr('title'),
|
||||
durationElement.text()?.trim()
|
||||
@@ -368,13 +451,18 @@ async function scrapeRumbleList({ q, page = 1, limit = 24, sort = 'viral' }) {
|
||||
|
||||
// Filtrage doublons par videoId (id ou "video/123...")
|
||||
const key = videoId;
|
||||
// Uploader : meilleur effort (nom de chaîne sous la carte)
|
||||
const uploaderName = card.find('.video-item--channel, .channel-name, a[href^="/c/"], a[href^="/user/"]').first().text().replace(/\s+/g, ' ').trim() || '';
|
||||
found.push({
|
||||
videoId: key,
|
||||
title,
|
||||
thumbnail: thumb,
|
||||
uploaderName: '',
|
||||
uploaderName,
|
||||
views,
|
||||
duration: durationSeconds,
|
||||
// Indicateur natif pour le front : une vidéo ≤ 75 s est un Short probable.
|
||||
// (Rumble n'expose pas de flag natif ; la durée reste la source de vérité.)
|
||||
isShort: durationSeconds > 0 && durationSeconds <= 75,
|
||||
uploadedDate: '',
|
||||
url,
|
||||
type: 'video'
|
||||
|
||||
Reference in New Issue
Block a user